diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-18 15:09:16 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-18 15:09:16 +0300 |
commit | adcc3955fe5226e808548d64b545ba94378e5d52 (patch) | |
tree | 4405a81119b4fd34cfd5321dba30a5fa613cfb13 | |
parent | 28724c880bbd3e1b5c9ef0d01e8856024c77ca6a (diff) |
Add latest changes from gitlab-org/gitlab@master
47 files changed, 800 insertions, 572 deletions
diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml index 229c4a99eca..b266435d4d2 100644 --- a/.rubocop_manual_todo.yml +++ b/.rubocop_manual_todo.yml @@ -44,7 +44,6 @@ Graphql/ResolverType: - 'app/graphql/resolvers/base_resolver.rb' - 'app/graphql/resolvers/ci/jobs_resolver.rb' - 'app/graphql/resolvers/ci/pipeline_stages_resolver.rb' - - 'app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb' - 'app/graphql/resolvers/merge_requests_resolver.rb' - 'app/graphql/resolvers/users/group_count_resolver.rb' - 'ee/app/graphql/resolvers/geo/merge_request_diff_registries_resolver.rb' diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 16e5c4e4738..092ab0d71d5 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -89c1ee804f273c9ccc7322644b9ec1cf8e38c0a4 +6373f0f90637d752a50a56c94edc57cc6e8a8566 diff --git a/app/assets/javascripts/integrations/integration_settings_form.js b/app/assets/javascripts/integrations/integration_settings_form.js index 1d0814125e6..14d6f133d27 100644 --- a/app/assets/javascripts/integrations/integration_settings_form.js +++ b/app/assets/javascripts/integrations/integration_settings_form.js @@ -35,12 +35,14 @@ export default class IntegrationSettingsForm { } saveIntegration() { - // Service was marked active so now we check; + // Save Service if not active and check the following if active; // 1) If form contents are valid // 2) If this service can be saved // If both conditions are true, we override form submission // and save the service using provided configuration. - if (this.$form.get(0).checkValidity()) { + const formValid = this.$form.get(0).checkValidity() || this.formActive === false; + + if (formValid) { this.$form.submit(); } else { eventHub.$emit('validateForm'); diff --git a/app/assets/javascripts/pages/groups/clusters/destroy/index.js b/app/assets/javascripts/pages/groups/clusters/destroy/index.js index 487e7a14a16..8001d2dd1da 100644 --- a/app/assets/javascripts/pages/groups/clusters/destroy/index.js +++ b/app/assets/javascripts/pages/groups/clusters/destroy/index.js @@ -1,3 +1,5 @@ import ClustersBundle from '~/clusters/clusters_bundle'; -new ClustersBundle(); // eslint-disable-line no-new +document.addEventListener('DOMContentLoaded', () => { + new ClustersBundle(); // eslint-disable-line no-new +}); diff --git a/app/assets/javascripts/pages/groups/clusters/edit/index.js b/app/assets/javascripts/pages/groups/clusters/edit/index.js index 487e7a14a16..8001d2dd1da 100644 --- a/app/assets/javascripts/pages/groups/clusters/edit/index.js +++ b/app/assets/javascripts/pages/groups/clusters/edit/index.js @@ -1,3 +1,5 @@ import ClustersBundle from '~/clusters/clusters_bundle'; -new ClustersBundle(); // eslint-disable-line no-new +document.addEventListener('DOMContentLoaded', () => { + new ClustersBundle(); // eslint-disable-line no-new +}); diff --git a/app/assets/javascripts/pages/groups/clusters/index.js b/app/assets/javascripts/pages/groups/clusters/index.js index 3b92c244346..9f466e0d60a 100644 --- a/app/assets/javascripts/pages/groups/clusters/index.js +++ b/app/assets/javascripts/pages/groups/clusters/index.js @@ -1,5 +1,7 @@ import initCreateCluster from '~/create_cluster/init_create_cluster'; import initIntegrationForm from '~/clusters/forms/show/index'; -initCreateCluster(document, gon); -initIntegrationForm(); +document.addEventListener('DOMContentLoaded', () => { + initCreateCluster(document, gon); + initIntegrationForm(); +}); diff --git a/app/assets/javascripts/pages/groups/clusters/index/index.js b/app/assets/javascripts/pages/groups/clusters/index/index.js index 3b71517f017..744be65bfbe 100644 --- a/app/assets/javascripts/pages/groups/clusters/index/index.js +++ b/app/assets/javascripts/pages/groups/clusters/index/index.js @@ -1,6 +1,8 @@ import PersistentUserCallout from '~/persistent_user_callout'; import initClustersListApp from '~/clusters_list'; -const callout = document.querySelector('.gcp-signup-offer'); -PersistentUserCallout.factory(callout); -initClustersListApp(); +document.addEventListener('DOMContentLoaded', () => { + const callout = document.querySelector('.gcp-signup-offer'); + PersistentUserCallout.factory(callout); + initClustersListApp(); +}); diff --git a/app/assets/javascripts/pages/groups/clusters/new/index.js b/app/assets/javascripts/pages/groups/clusters/new/index.js index de9ded87ef3..876bab0b339 100644 --- a/app/assets/javascripts/pages/groups/clusters/new/index.js +++ b/app/assets/javascripts/pages/groups/clusters/new/index.js @@ -1,3 +1,5 @@ import initNewCluster from '~/clusters/new_cluster'; -initNewCluster(); +document.addEventListener('DOMContentLoaded', () => { + initNewCluster(); +}); diff --git a/app/assets/javascripts/pages/groups/clusters/show/index.js b/app/assets/javascripts/pages/groups/clusters/show/index.js index 5d202a8824f..ccf631b2c53 100644 --- a/app/assets/javascripts/pages/groups/clusters/show/index.js +++ b/app/assets/javascripts/pages/groups/clusters/show/index.js @@ -1,5 +1,7 @@ import ClustersBundle from '~/clusters/clusters_bundle'; import initClusterHealth from '~/pages/projects/clusters/show/cluster_health'; -new ClustersBundle(); // eslint-disable-line no-new -initClusterHealth(); +document.addEventListener('DOMContentLoaded', () => { + new ClustersBundle(); // eslint-disable-line no-new + initClusterHealth(); +}); diff --git a/app/assets/javascripts/pages/projects/commit/pipelines/index.js b/app/assets/javascripts/pages/projects/commit/pipelines/index.js index 26dea17ca8a..eaf340f2725 100644 --- a/app/assets/javascripts/pages/projects/commit/pipelines/index.js +++ b/app/assets/javascripts/pages/projects/commit/pipelines/index.js @@ -1,8 +1,5 @@ import { initCommitBoxInfo } from '~/projects/commit_box/info'; import initPipelines from '~/commit/pipelines/pipelines_bundle'; -document.addEventListener('DOMContentLoaded', () => { - initCommitBoxInfo(); - - initPipelines(); -}); +initCommitBoxInfo(); +initPipelines(); diff --git a/app/assets/javascripts/pages/projects/commit/show/index.js b/app/assets/javascripts/pages/projects/commit/show/index.js index e0bd49bf6ef..0750f472341 100644 --- a/app/assets/javascripts/pages/projects/commit/show/index.js +++ b/app/assets/javascripts/pages/projects/commit/show/index.js @@ -15,35 +15,33 @@ import { __ } from '~/locale'; import loadAwardsHandler from '~/awards_handler'; import { initCommitBoxInfo } from '~/projects/commit_box/info'; -document.addEventListener('DOMContentLoaded', () => { - const hasPerfBar = document.querySelector('.with-performance-bar'); - const performanceHeight = hasPerfBar ? 35 : 0; - initChangesDropdown(document.querySelector('.navbar-gitlab').offsetHeight + performanceHeight); - new ZenMode(); - new ShortcutsNavigation(); +const hasPerfBar = document.querySelector('.with-performance-bar'); +const performanceHeight = hasPerfBar ? 35 : 0; +initChangesDropdown(document.querySelector('.navbar-gitlab').offsetHeight + performanceHeight); +new ZenMode(); +new ShortcutsNavigation(); - initCommitBoxInfo(); +initCommitBoxInfo(); - initNotes(); +initNotes(); - const filesContainer = $('.js-diffs-batch'); +const filesContainer = $('.js-diffs-batch'); - if (filesContainer.length) { - const batchPath = filesContainer.data('diffFilesPath'); +if (filesContainer.length) { + const batchPath = filesContainer.data('diffFilesPath'); - axios - .get(batchPath) - .then(({ data }) => { - filesContainer.html($(data.html)); - syntaxHighlight(filesContainer); - handleLocationHash(); - new Diff(); - }) - .catch(() => { - flash({ message: __('An error occurred while retrieving diff files') }); - }); - } else { - new Diff(); - } - loadAwardsHandler(); -}); + axios + .get(batchPath) + .then(({ data }) => { + filesContainer.html($(data.html)); + syntaxHighlight(filesContainer); + handleLocationHash(); + new Diff(); + }) + .catch(() => { + flash({ message: __('An error occurred while retrieving diff files') }); + }); +} else { + new Diff(); +} +loadAwardsHandler(); diff --git a/app/assets/javascripts/pages/projects/commits/show/index.js b/app/assets/javascripts/pages/projects/commits/show/index.js index b456baac612..6239e4c99d2 100644 --- a/app/assets/javascripts/pages/projects/commits/show/index.js +++ b/app/assets/javascripts/pages/projects/commits/show/index.js @@ -1,12 +1,9 @@ import CommitsList from '~/commits'; import GpgBadges from '~/gpg_badges'; import ShortcutsNavigation from '~/behaviors/shortcuts/shortcuts_navigation'; - import mountCommits from '~/projects/commits'; -document.addEventListener('DOMContentLoaded', () => { - new CommitsList(document.querySelector('.js-project-commits-show').dataset.commitsLimit); // eslint-disable-line no-new - new ShortcutsNavigation(); // eslint-disable-line no-new - GpgBadges.fetch(); - mountCommits(document.getElementById('js-author-dropdown')); -}); +new CommitsList(document.querySelector('.js-project-commits-show').dataset.commitsLimit); // eslint-disable-line no-new +new ShortcutsNavigation(); // eslint-disable-line no-new +GpgBadges.fetch(); +mountCommits(document.getElementById('js-author-dropdown')); diff --git a/app/assets/javascripts/pages/projects/settings/access_tokens/index.js b/app/assets/javascripts/pages/projects/settings/access_tokens/index.js index ae2209b0292..22dddb72f98 100644 --- a/app/assets/javascripts/pages/projects/settings/access_tokens/index.js +++ b/app/assets/javascripts/pages/projects/settings/access_tokens/index.js @@ -1,3 +1,3 @@ import initExpiresAtField from '~/access_tokens'; -document.addEventListener('DOMContentLoaded', initExpiresAtField); +initExpiresAtField(); diff --git a/app/assets/javascripts/pages/projects/settings/repository/create_deploy_token/index.js b/app/assets/javascripts/pages/projects/settings/repository/create_deploy_token/index.js index ffc84dc106b..1dc238b56b4 100644 --- a/app/assets/javascripts/pages/projects/settings/repository/create_deploy_token/index.js +++ b/app/assets/javascripts/pages/projects/settings/repository/create_deploy_token/index.js @@ -1,3 +1,3 @@ import initForm from '../form'; -document.addEventListener('DOMContentLoaded', initForm); +initForm(); diff --git a/app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb b/app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb index 669b487db10..13b5672d750 100644 --- a/app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb +++ b/app/graphql/resolvers/error_tracking/sentry_error_stack_trace_resolver.rb @@ -3,6 +3,8 @@ module Resolvers module ErrorTracking class SentryErrorStackTraceResolver < BaseResolver + type Types::ErrorTracking::SentryErrorStackTraceType, null: true + argument :id, ::Types::GlobalIDType[::Gitlab::ErrorTracking::DetailedError], required: true, description: 'ID of the Sentry issue' diff --git a/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb b/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb index c5cf924ce7f..e844ffedbeb 100644 --- a/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb +++ b/app/graphql/resolvers/error_tracking/sentry_errors_resolver.rb @@ -4,19 +4,26 @@ module Resolvers module ErrorTracking class SentryErrorsResolver < BaseResolver type Types::ErrorTracking::SentryErrorType.connection_type, null: true + extension Gitlab::Graphql::Extensions::ExternallyPaginatedArrayExtension + + argument :search_term, ::GraphQL::STRING_TYPE, + description: 'Search query for the Sentry error details', + required: false + + # TODO: convert to Enum + argument :sort, ::GraphQL::STRING_TYPE, + description: 'Attribute to sort on. Options are frequency, first_seen, last_seen. last_seen is default', + required: false + + delegate :project, to: :object def resolve(**args) args[:cursor] = args.delete(:after) - project = object.project - result = ::ErrorTracking::ListIssuesService.new( - project, - context[:current_user], - args - ).execute + result = ::ErrorTracking::ListIssuesService.new(project, current_user, args).execute - next_cursor = result[:pagination]&.dig('next', 'cursor') - previous_cursor = result[:pagination]&.dig('previous', 'cursor') + next_cursor = result.dig(:pagination, 'next', 'cursor') + previous_cursor = result.dig(:pagination, 'previous', 'cursor') issues = result[:issues] # ReactiveCache is still fetching data @@ -24,6 +31,10 @@ module Resolvers Gitlab::Graphql::ExternallyPaginatedArray.new(previous_cursor, next_cursor, *issues) end + + def self.field_options + super.merge(connection: false) # we manage the pagination manually, so opt out of the connection field extension + end end end end diff --git a/app/graphql/types/error_tracking/sentry_error_collection_type.rb b/app/graphql/types/error_tracking/sentry_error_collection_type.rb index 798e0433d06..49d5d62c860 100644 --- a/app/graphql/types/error_tracking/sentry_error_collection_type.rb +++ b/app/graphql/types/error_tracking/sentry_error_collection_type.rb @@ -9,27 +9,12 @@ module Types authorize :read_sentry_issue field :errors, - Types::ErrorTracking::SentryErrorType.connection_type, - connection: false, - null: true, description: "Collection of Sentry Errors", - extensions: [Gitlab::Graphql::Extensions::ExternallyPaginatedArrayExtension], - resolver: Resolvers::ErrorTracking::SentryErrorsResolver do - argument :search_term, - String, - description: 'Search query for the Sentry error details', - required: false - argument :sort, - String, - description: 'Attribute to sort on. Options are frequency, first_seen, last_seen. last_seen is default', - required: false - end - field :detailed_error, Types::ErrorTracking::SentryDetailedErrorType, - null: true, + resolver: Resolvers::ErrorTracking::SentryErrorsResolver + field :detailed_error, description: 'Detailed version of a Sentry error on the project', resolver: Resolvers::ErrorTracking::SentryDetailedErrorResolver - field :error_stack_trace, Types::ErrorTracking::SentryErrorStackTraceType, - null: true, + field :error_stack_trace, description: 'Stack Trace of Sentry Error', resolver: Resolvers::ErrorTracking::SentryErrorStackTraceResolver field :external_url, diff --git a/changelogs/unreleased/44933-fix-unreachable-cli-image-openshift-ci-template.yml b/changelogs/unreleased/44933-fix-unreachable-cli-image-openshift-ci-template.yml new file mode 100644 index 00000000000..1be7606e3b4 --- /dev/null +++ b/changelogs/unreleased/44933-fix-unreachable-cli-image-openshift-ci-template.yml @@ -0,0 +1,5 @@ +--- +title: Fix the unreachable CLI image in OpenShift CI template +merge_request: 44933 +author: Klaus Mueller @klml +type: added diff --git a/changelogs/unreleased/frontend-validate-only-active-project-services.yml b/changelogs/unreleased/frontend-validate-only-active-project-services.yml new file mode 100644 index 00000000000..8732b32330c --- /dev/null +++ b/changelogs/unreleased/frontend-validate-only-active-project-services.yml @@ -0,0 +1,5 @@ +--- +title: Fix project integration form validation when integration is inactive +merge_request: 47201 +author: +type: fixed diff --git a/changelogs/unreleased/reference-cs-image-in-template.yml b/changelogs/unreleased/reference-cs-image-in-template.yml new file mode 100644 index 00000000000..161607e5607 --- /dev/null +++ b/changelogs/unreleased/reference-cs-image-in-template.yml @@ -0,0 +1,5 @@ +--- +title: Use CS_ANALYZER_IMAGE in CS template +merge_request: 47856 +author: +type: added diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 02b907ae237..fdea46c30ac 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -208,7 +208,9 @@ Omnibus GitLab-managed database. External databases are currently not supported. In some circumstances, like during [upgrades](replication/updating_the_geo_nodes.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. -Pausing and resuming replication is done via a command line tool from the secondary node. +Pausing and resuming replication is done via a command line tool from the secondary node where the `postgresql` service is enabled. + +If `postgresql` is on a standalone database node, ensure that `gitlab.rb` on that node contains the configuration line `gitlab_rails['geo_node_name'] = 'node_name'`, where `node_name` is the same as the `geo_name_name` on the application node. **To Pause: (from secondary)** diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 7d65d2165c5..9d5653bcc72 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -174,6 +174,12 @@ the **primary** database. Use the following as a guide. roles ['geo_secondary_role', 'postgres_role'] ## + ## The unique identifier for the Geo node. + ## This should match the secondary's application node. + ## + gitlab_rails['geo_node_name'] = '<node_name_here>' + + ## ## Secondary address ## - replace '<secondary_node_ip>' with the public or VPC address of your Geo secondary node ## - replace '<tracking_database_ip>' with the public or VPC address of your Geo tracking database node diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 8816d0eecf4..54f3eca204b 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -151,7 +151,27 @@ is recommended. instance to other geographical locations as a read-only fully operational instance that can also be promoted in case of disaster. -## Configuring select components with Cloud Native Helm +## Deviating from the suggested reference architectures + +As a general rule of thumb, the further away you move from the Reference Architectures, +the harder it will be get support for it. With any deviation, you're introducing +a layer of complexity that will add challenges to finding out where potential +issues might lie. + +The reference architectures use the official GitLab Linux packages (Omnibus +GitLab) to install and configure the various components (with one notable exception being the suggested select Cloud Native installation method described below). The components are +installed on separate machines (virtualized or bare metal), with machine hardware +requirements listed in the "Configuration" column and equivalent VM standard sizes listed +in GCP/AWS/Azure columns of each [available reference architecture](#available-reference-architectures). + +Running components on Docker (including Compose) with the same specs should be fine, as Docker is well known in terms of support. +However, it is still an additional layer and may still add some support complexities, such as not being able to run `strace` easily in containers. + +Other technologies, like [Docker swarm](https://docs.docker.com/engine/swarm/) +are not officially supported, but can be implemented at your own risk. In that +case, GitLab Support will not be able to help you. + +### Configuring select components with Cloud Native Helm We also provide [Helm charts](https://docs.gitlab.com/charts/) as a Cloud Native installation method for GitLab. For the reference architectures, select components can be set up in this diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index de3f9c2665f..22fe96222ef 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -57602,8 +57602,8 @@ "description": "Collection of Sentry Errors", "args": [ { - "name": "after", - "description": "Returns the elements in the list that come after the specified cursor.", + "name": "searchTerm", + "description": "Search query for the Sentry error details", "type": { "kind": "SCALAR", "name": "String", @@ -57612,8 +57612,8 @@ "defaultValue": null }, { - "name": "before", - "description": "Returns the elements in the list that come before the specified cursor.", + "name": "sort", + "description": "Attribute to sort on. Options are frequency, first_seen, last_seen. last_seen is default", "type": { "kind": "SCALAR", "name": "String", @@ -57622,41 +57622,41 @@ "defaultValue": null }, { - "name": "first", - "description": "Returns the first _n_ elements from the list.", + "name": "after", + "description": "Returns the elements in the list that come after the specified cursor.", "type": { "kind": "SCALAR", - "name": "Int", + "name": "String", "ofType": null }, "defaultValue": null }, { - "name": "last", - "description": "Returns the last _n_ elements from the list.", + "name": "before", + "description": "Returns the elements in the list that come before the specified cursor.", "type": { "kind": "SCALAR", - "name": "Int", + "name": "String", "ofType": null }, "defaultValue": null }, { - "name": "searchTerm", - "description": "Search query for the Sentry error details", + "name": "first", + "description": "Returns the first _n_ elements from the list.", "type": { "kind": "SCALAR", - "name": "String", + "name": "Int", "ofType": null }, "defaultValue": null }, { - "name": "sort", - "description": "Attribute to sort on. Options are frequency, first_seen, last_seen. last_seen is default", + "name": "last", + "description": "Returns the last _n_ elements from the list.", "type": { "kind": "SCALAR", - "name": "String", + "name": "Int", "ofType": null }, "defaultValue": null diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index 27b76d1f0c0..0ad3528fa19 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -20,9 +20,9 @@ GET /groups/:id/clusters Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ----------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | Example request: @@ -39,6 +39,8 @@ Example response: "name":"cluster-1", "domain":"example.com", "created_at":"2019-01-02T20:18:12.563Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -87,10 +89,10 @@ GET /groups/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | -------------- | -------- | ----------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | Example request: @@ -106,6 +108,8 @@ Example response: "name":"cluster-1", "domain":"example.com", "created_at":"2019-01-02T20:18:12.563Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -154,19 +158,19 @@ POST /groups/:id/clusters/user Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `name` | string | yes | The name of the cluster | -| `domain` | string | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | -| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | -| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | -| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | +| Attribute | Type | Required | Description | +| ---------------------------------------------------- | -------------- | -------- | --------------------------------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `name` | string | yes | The name of the cluster | +| `domain` | string | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | +| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | +| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | Example request: @@ -184,6 +188,8 @@ Example response: "id":24, "name":"cluster-5", "created_at":"2019-01-03T21:53:40.610Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -223,17 +229,19 @@ PUT /groups/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `cluster_id` | integer | yes | The ID of the cluster | -| `name` | string | no | The name of the cluster | -| `domain` | string | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** | +| Attribute | Type | Required | Description | +| ----------------------------------------- | -------------- | -------- | ------------------------------------------------------------------------------------------ | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | +| `name` | string | no | The name of the cluster | +| `domain` | string | no | The [base domain](../user/group/clusters/index.md#base-domain) of the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster | +| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** | NOTE: **Note:** `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added @@ -256,6 +264,8 @@ Example response: "name":"new-cluster-name", "domain":"new-domain.com", "created_at":"2019-01-03T21:53:40.610Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -304,10 +314,10 @@ DELETE /groups/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | -------------- | -------- | ----------------------------------------------------------------------------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) | +| `cluster_id` | integer | yes | The ID of the cluster | Example request: diff --git a/doc/api/instance_clusters.md b/doc/api/instance_clusters.md index bc4eca5abfd..f57a61474dc 100644 --- a/doc/api/instance_clusters.md +++ b/doc/api/instance_clusters.md @@ -35,6 +35,8 @@ Example response: "id": 9, "name": "cluster-1", "created_at": "2020-07-14T18:36:10.440Z", + "managed": true, + "enabled": true, "domain": null, "provider_type": "user", "platform_type": "kubernetes", @@ -98,9 +100,9 @@ Returns a single instance cluster. Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------- | +| `cluster_id` | integer | yes | The ID of the cluster | ```plaintext GET /admin/clusters/:cluster_id @@ -119,6 +121,8 @@ Example response: "id": 9, "name": "cluster-1", "created_at": "2020-07-14T18:36:10.440Z", + "managed": true, + "enabled": true, "domain": null, "provider_type": "user", "platform_type": "kubernetes", @@ -153,19 +157,19 @@ POST /admin/clusters/add Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `name` | string | yes | The name of the cluster | -| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | -| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | -| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | -| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | -| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | +| Attribute | Type | Required | Description | +| ---------------------------------------------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------- | +| `name` | string | yes | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not, defaults to `true` | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to `true` | +| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | Example request: @@ -184,6 +188,8 @@ Example response: "id": 11, "name": "cluster-3", "created_at": "2020-07-14T18:42:50.805Z", + "managed": true, + "enabled": true, "domain": null, "provider_type": "user", "platform_type": "kubernetes", @@ -218,18 +224,19 @@ PUT /admin/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `cluster_id` | integer | yes | The ID of the cluster | -| `name` | string | no | The name of the cluster | -| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | -| `environment_scope` | string | no | The associated environment to the cluster | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | -| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| Attribute | Type | Required | Description | +| ------------------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------ | +| `cluster_id` | integer | yes | The ID of the cluster | +| `name` | string | no | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `environment_scope` | string | no | The associated environment to the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster | +| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | NOTE: **Note:** `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added @@ -252,6 +259,8 @@ Example response: "id": 9, "name": "update-cluster-name", "created_at": "2020-07-14T18:36:10.440Z", + "managed": true, + "enabled": true, "domain": null, "provider_type": "user", "platform_type": "kubernetes", @@ -288,9 +297,9 @@ DELETE /admin/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------- | +| `cluster_id` | integer | yes | The ID of the cluster | Example request: diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index ce175184179..44b16e188af 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -20,9 +20,9 @@ GET /projects/:id/clusters Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- | ----------------------------------------------------- | +| `id` | integer | yes | The ID of the project owned by the authenticated user | Example request: @@ -39,6 +39,8 @@ Example response: "name":"cluster-1", "domain":"example.com", "created_at":"2019-01-02T20:18:12.563Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -88,10 +90,10 @@ GET /projects/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ----------------------------------------------------- | +| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `cluster_id` | integer | yes | The ID of the cluster | Example request: @@ -107,6 +109,8 @@ Example response: "name":"cluster-1", "domain":"example.com", "created_at":"2019-01-02T20:18:12.563Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -179,20 +183,20 @@ POST /projects/:id/clusters/user Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | -| `name` | string | yes | The name of the cluster | -| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | -| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | -| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | -| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | -| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | +| Attribute | Type | Required | Description | +| ---------------------------------------------------- | ------- | -------- | ----------------------------------------------------------------------------------------------------- | +| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `name` | string | yes | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not, defaults to `true` | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to `true` | +| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | +| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | Example request: @@ -210,6 +214,8 @@ Example response: "id":24, "name":"cluster-5", "created_at":"2019-01-03T21:53:40.610Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -273,18 +279,20 @@ PUT /projects/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | -| `cluster_id` | integer | yes | The ID of the cluster | -| `name` | string | no | The name of the cluster | -| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | -| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | -| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | -| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** | +| Attribute | Type | Required | Description | +| ------------------------------------------- | ------- | -------- | ------------------------------------------------------------------------------------------ | +| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `cluster_id` | integer | yes | The ID of the cluster | +| `name` | string | no | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | +| `enabled` | boolean | no | Determines if cluster is active or not | +| `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster | +| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** | NOTE: **Note:** `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added @@ -307,6 +315,8 @@ Example response: "name":"new-cluster-name", "domain":"new-domain.com", "created_at":"2019-01-03T21:53:40.610Z", + "managed": true, + "enabled": true, "provider_type":"user", "platform_type":"kubernetes", "environment_scope":"*", @@ -380,10 +390,10 @@ DELETE /projects/:id/clusters/:cluster_id Parameters: -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer | yes | The ID of the project owned by the authenticated user | -| `cluster_id` | integer | yes | The ID of the cluster | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ----------------------------------------------------- | +| `id` | integer | yes | The ID of the project owned by the authenticated user | +| `cluster_id` | integer | yes | The ID of the cluster | Example request: diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 19159c6c0ff..3ab865170ae 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -59,6 +59,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [Client-side connection-pool](client_side_connection_pool.md) - [Updating multiple values](setting_multiple_values.md) - [Constraints naming conventions](constraint_naming_convention.md) +- [Query performance guidelines](../query_performance.md) ## Case studies diff --git a/doc/development/database_review.md b/doc/development/database_review.md index d1ec32af464..5ae0c25c3b5 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -158,8 +158,8 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac - Maintainer: After the merge request is merged, notify Release Managers about it on `#f_upcoming_release` Slack channel. - Check consistency with `db/structure.sql` and that migrations are [reversible](migration_style_guide.md#reversibility) - Check that the relevant version files under `db/schema_migrations` were added or removed. - - Check queries timing (If any): Queries executed in a migration - need to fit comfortably within `15s` - preferably much less than that - on GitLab.com. + - Check queries timing (If any): In a single transaction, cumulative query time executed in a migration + needs to fit comfortably within `15s` - preferably much less than that - on GitLab.com. - For column removals, make sure the column has been [ignored in a previous release](what_requires_downtime.md#dropping-columns) - Check [background migrations](background_migrations.md): - Establish a time estimate for execution on GitLab.com. For historical purposes, @@ -190,7 +190,7 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac - For given queries, review parameters regarding data distribution - [Check query plans](understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) - - General guideline is for queries to come in below 100ms execution time + - General guideline is for queries to come in below [100ms execution time](query_performance.md#timing-guidelines-for-queries) - Avoid N+1 problems and minimalize the [query count](merge_request_performance_guidelines.md#query-counts). ### Timing guidelines for migrations @@ -206,4 +206,4 @@ Keep in mind that all runtimes should be measured against GitLab.com. |----|----|---| | Regular migrations on `db/migrate` | `3 minutes` | A valid exception are index creation as this can take a long time. | | Post migrations on `db/post_migrate` | `10 minutes` | | -| Background migrations | --- | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below `1 second` execution time with cold caches. | +| Background migrations | --- | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below [`1 second` execution time](query_performance.md#timing-guidelines-for-queries) with cold caches. | diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 84679a78545..78acd729b9b 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -153,8 +153,9 @@ and therefore it does not have any records yet. When using a single-transaction migration, a transaction will hold on a database connection for the duration of the migration, so you must make sure the actions in the migration -do not take too much time: In general, queries executed in a migration need to fit comfortably -within `15s` on GitLab.com. +do not take too much time: GitLab.com’s production database has a `15s` timeout, so +in general, the cumulative execution time in a migration should aim to fit comfortably +in that limit. Singular query timings should fit within the [standard limit](query_performance.md#timing-guidelines-for-queries) In case you need to insert, update, or delete a significant amount of data, you: diff --git a/doc/development/product_analytics/usage_ping.md b/doc/development/product_analytics/usage_ping.md index fa785d934cb..893814c5f0d 100644 --- a/doc/development/product_analytics/usage_ping.md +++ b/doc/development/product_analytics/usage_ping.md @@ -631,7 +631,7 @@ Paste the SQL query into `#database-lab` to see how the query performs at scale. - `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. - GitLab.com’s production database has a 15 second timeout. -- Any single query must stay below 1 second execution time with cold caches. +- Any single query must stay below [1 second execution time](../query_performance.md#timing-guidelines-for-queries) with cold caches. - Add a specialized index on columns involved to reduce the execution time. In order to have an understanding of the query's execution we add in the MR description the following information: diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md new file mode 100644 index 00000000000..b8be2bff07f --- /dev/null +++ b/doc/development/query_performance.md @@ -0,0 +1,74 @@ +--- +stage: Enablement +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +--- + +# Query performance guidelines + +This document describes various guidelines to follow when optimizing SQL queries. + +When you are optimizing your SQL queries, there are two dimensions to pay attention to: + +1. The query execution time. This is paramount as it reflects how the user experiences GitLab. +1. The query plan. Optimizing the query plan is important in allowing queries to independently scale over time. Realizing that an index will keep a query performing well as the table grows before the query degrades is an example of why we analyze these plans. + +## Timing guidelines for queries + +| Query Type | Maximum Query Time | Notes | +|----|----|---| +| General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. | +| Queries in a migration | `100ms` | This is different than the total [migration time](database_review.md#timing-guidelines-for-migrations). | +| Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | +| Background migrations | `1s` | | +| Usage Ping | `1s` | See the [usage ping docs](product_analytics/usage_ping.md#developing-and-testing-usage-ping) for more details. | + +- When analyzing your query's performance, pay attention to if the time you are seeing is on a [cold or warm cache](#cold-and-warm-cache). These guidelines apply for both cache types. +- When working with batched queries, change the range and batch size to see how it effects the query timing and caching. +- If an existing query is already underperforming, make an effort to improve it. If it is too complex or would stall development, create a follow-up so it can be addressed in a timely manner. You can always ask the database reviewer or maintainer for help and guidance. + +## Cold and warm cache + +When evaluating query performance it is important to understand the difference between +cold and warm cached queries. + +The first time a query is made, it is made on a "cold cache". Meaning it needs +to read from disk. If you run the query again, the data can be read from the +cache, or what PostgreSQL calls shared buffers. This is the "warm cache" query. + +When analyzing an [`EXPLAIN` plan](understanding_explain_plans.md), you can see +the difference not only in the timing, but by looking at the output for `Buffers` +by running your explain with `EXPLAIN(analyze, buffers)`. The [#database-lab](understanding_explain_plans.md#database-lab) +tool will automatically include these options. + +If you are making a warm cache query, you will only see the `shared hits`. + +For example in #database-lab: + +```plaintext +Shared buffers: + - hits: 36467 (~284.90 MiB) from the buffer pool + - reads: 0 from the OS file cache, including disk I/O +``` + +Or in the explain plan from `psql`: + +```sql +Buffers: shared hit=7323 +``` + +If the cache is cold, you will also see `reads`. + +In #database-lab: + +```plaintext +Shared buffers: + - hits: 17204 (~134.40 MiB) from the buffer pool + - reads: 15229 (~119.00 MiB) from the OS file cache, including disk I/O +``` + +In `psql`: + +```sql +Buffers: shared hit=7202 read=121 +``` diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index e58667c275c..c2d4c77a469 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -66,10 +66,12 @@ easily [edit your content](#edit-content). 1. To get started, create a new project from the [Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) template. You can either [fork it](../repository/forking_workflow.md#creating-a-fork) or [create a new project from a template](../../../gitlab-basics/create-project.md#built-in-templates). -1. Edit the [`data/config.yml`](#configuration-files) configuration file +1. Edit the [`data/config.yml`](#static-site-generator-configuration) configuration file to replace `<username>` and `<project-name>` with the proper values for - your project's path. This triggers a CI/CD pipeline to deploy your project - with GitLab Pages. + your project's path. +1. (Optional) Edit the [`.gitlab/static-site-editor.yml`](#static-site-editor-configuration-file) file + to customize the behavior of the Static Site Editor. +1. When you submit your changes, GitLab triggers a CI/CD pipeline to deploy your project with GitLab Pages. 1. When the pipeline finishes, from your project's left-side menu, go to **Settings > Pages** to find the URL of your new website. 1. Visit your website and look at the bottom-left corner of the screen to see the new **Edit this page** button. @@ -179,6 +181,41 @@ yet. You can do so by editing the file locally, through the GitLab regular file ## Configuration files +You can customize the behavior of a project which uses the Static Site Editor with +the following configuration files: + +- The [`.gitlab/static-site-editor.yml`](#static-site-editor-configuration-file), which customizes the + behavior of the Static Site Editor. +- [Static Site Generator configuration files](#static-site-generator-configuration), + such as `data/config.yml`, which configures the Static Site Generator itself. + It also controls the **Edit this page** button when the site is generated. + +### Static Site Editor configuration file + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4267) in GitLab 13.6. + +The `.gitlab/static-site-editor.yml` configuration file contains entries you can +use to customize behavior of the Static Site Editor (SSE). If the file does not exist, +default values which support a default Middleman project configuration are used. +The [Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) project template generates a file pre-populated with these defaults. + +To customize the behavior of the SSE, edit `.gitlab/static-site-editor.yml`'s entries +(described in the table below) according to what works best for your project (respecting YAML syntax). + +After the table, see an [example of the SSE configuration file](#gitlabstatic-site-editoryml-example). + +| Entry | GitLab version | Type | Default value | Description | +|---|---|---|---|---| +| `image_upload_path` | [13.6](https://gitlab.com/gitlab-org/gitlab/-/issues/216641) | String | `source/images` | Directory for images uploaded from the WYSIWYG editor. | + +#### `.gitlab/static-site-editor.yml` example + +```yaml +image_upload_path: 'source/images' # Relative path to the project's root. Don't include leading or trailing slashes. +``` + +### Static Site Generator configuration + The Static Site Editor uses Middleman's configuration file, `data/config.yml` to customize the behavior of the project itself and to control the **Edit this page** button, rendered through the file [`layout.erb`](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/-/blob/master/source/layouts/layout.erb). diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 679e231b283..b724d3a38dc 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -76,6 +76,7 @@ module API optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb index 08265201328..8c13b580092 100644 --- a/lib/api/conan_instance_packages.rb +++ b/lib/api/conan_instance_packages.rb @@ -4,7 +4,7 @@ module API class ConanInstancePackages < ::API::Base namespace 'packages/conan/v1' do - include ConanPackageEndpoints + include ::API::Concerns::Packages::ConanEndpoints end end end diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb deleted file mode 100644 index 188a42f26f8..00000000000 --- a/lib/api/conan_package_endpoints.rb +++ /dev/null @@ -1,351 +0,0 @@ -# frozen_string_literal: true - -# Conan Package Manager Client API -# -# These API endpoints are not consumed directly by users, so there is no documentation for the -# individual endpoints. They are called by the Conan package manager client when users run commands -# like `conan install` or `conan upload`. The usage of the GitLab Conan repository is documented here: -# https://docs.gitlab.com/ee/user/packages/conan_repository/#installing-a-package -# -# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 -module API - module ConanPackageEndpoints - extend ActiveSupport::Concern - - PACKAGE_REQUIREMENTS = { - package_name: API::NO_SLASH_URL_PART_REGEX, - package_version: API::NO_SLASH_URL_PART_REGEX, - package_username: API::NO_SLASH_URL_PART_REGEX, - package_channel: API::NO_SLASH_URL_PART_REGEX - }.freeze - - FILE_NAME_REQUIREMENTS = { - file_name: API::NO_SLASH_URL_PART_REGEX - }.freeze - - PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex - CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex - - CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze - - included do - feature_category :package_registry - - helpers ::API::Helpers::PackagesManagerClientsHelpers - helpers ::API::Helpers::Packages::Conan::ApiHelpers - helpers ::API::Helpers::RelatedResourcesHelpers - - before do - require_packages_enabled! - - # Personal access token will be extracted from Bearer or Basic authorization - # in the overridden find_personal_access_token or find_user_from_job_token helpers - authenticate! - end - - desc 'Ping the Conan API' do - detail 'This feature was introduced in GitLab 12.2' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'ping' do - header 'X-Conan-Server-Capabilities', [].join(',') - end - - desc 'Search for packages' do - detail 'This feature was introduced in GitLab 12.4' - end - - params do - requires :q, type: String, desc: 'Search query' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'conans/search' do - service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute - service.payload - end - - namespace 'users' do - format :txt - - desc 'Authenticate user against conan CLI' do - detail 'This feature was introduced in GitLab 12.2' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'authenticate' do - unauthorized! unless token - - token.to_jwt - end - - desc 'Check for valid user credentials per conan CLI' do - detail 'This feature was introduced in GitLab 12.4' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'check_credentials' do - authenticate! - :ok - end - end - - params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' - end - namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do - # Get the snapshot - # - # the snapshot is a hash of { filename: md5 hash } - # md5 hash is the has of that file. This hash is used to diff the files existing on the client - # to determine which client files need to be uploaded if no recipe exists the snapshot is empty - desc 'Package Snapshot' do - detail 'This feature was introduced in GitLab 12.5' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference' do - authorize!(:read_package, project) - - presenter = ::Packages::Conan::PackagePresenter.new( - package, - current_user, - project, - conan_package_reference: params[:conan_package_reference] - ) - - present presenter, with: ::API::Entities::ConanPackage::ConanPackageSnapshot - end - - desc 'Recipe Snapshot' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - authorize!(:read_package, project) - - presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) - - present presenter, with: ::API::Entities::ConanPackage::ConanRecipeSnapshot - end - - # Get the manifest - # returns the download urls for the existing recipe in the registry - # - # the manifest is a hash of { filename: url } - # where the url is the download url for the file - desc 'Package Digest' do - detail 'This feature was introduced in GitLab 12.5' - end - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference/digest' do - present_package_download_urls - end - - desc 'Recipe Digest' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'digest' do - present_recipe_download_urls - end - - # Get the download urls - # - # returns the download urls for the existing recipe or package in the registry - # - # the manifest is a hash of { filename: url } - # where the url is the download url for the file - desc 'Package Download Urls' do - detail 'This feature was introduced in GitLab 12.5' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference/download_urls' do - present_package_download_urls - end - - desc 'Recipe Download Urls' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'download_urls' do - present_recipe_download_urls - end - - # Get the upload urls - # - # request body contains { filename: filesize } where the filename is the - # name of the file the conan client is requesting to upload - # - # returns { filename: url } - # where the url is the upload url for the file that the conan client will use - desc 'Package Upload Urls' do - detail 'This feature was introduced in GitLab 12.4' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - post 'packages/:conan_package_reference/upload_urls' do - authorize!(:read_package, project) - - status 200 - present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls - end - - desc 'Recipe Upload Urls' do - detail 'This feature was introduced in GitLab 12.4' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - post 'upload_urls' do - authorize!(:read_package, project) - - status 200 - present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls - end - - desc 'Delete Package' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - delete do - authorize!(:destroy_package, project) - - track_package_event('delete_package', :conan, category: 'API::ConanPackages') - - package.destroy - end - end - - params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' - requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' - end - namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do - before do - authenticate_non_get! - end - - params do - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES - end - namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do - desc 'Download recipe files' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - download_package_file(:recipe_file) - end - - desc 'Upload recipe package files' do - detail 'This feature was introduced in GitLab 12.6' - end - - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put do - upload_package_file(:recipe_file) - end - - desc 'Workhorse authorize the conan recipe file' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put 'authorize' do - authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) - end - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan Package ID' - requires :package_revision, type: String, desc: 'Conan Package Revision' - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES - end - namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do - desc 'Download package files' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - download_package_file(:package_file) - end - - desc 'Workhorse authorize the conan package file' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put 'authorize' do - authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) - end - - desc 'Upload package files' do - detail 'This feature was introduced in GitLab 12.6' - end - - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put do - upload_package_file(:package_file) - end - end - end - end - end -end diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb index db8cd187811..636b5dca5ed 100644 --- a/lib/api/conan_project_packages.rb +++ b/lib/api/conan_project_packages.rb @@ -9,7 +9,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/conan/v1' do - include ConanPackageEndpoints + include ::API::Concerns::Packages::ConanEndpoints end end end diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb new file mode 100644 index 00000000000..6c8b3a1ba4a --- /dev/null +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -0,0 +1,355 @@ +# frozen_string_literal: true + +# Conan Package Manager Client API +# +# These API endpoints are not consumed directly by users, so there is no documentation for the +# individual endpoints. They are called by the Conan package manager client when users run commands +# like `conan install` or `conan upload`. The usage of the GitLab Conan repository is documented here: +# https://docs.gitlab.com/ee/user/packages/conan_repository/#installing-a-package +# +# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 +module API + module Concerns + module Packages + module ConanEndpoints + extend ActiveSupport::Concern + + PACKAGE_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX, + package_version: API::NO_SLASH_URL_PART_REGEX, + package_username: API::NO_SLASH_URL_PART_REGEX, + package_channel: API::NO_SLASH_URL_PART_REGEX + }.freeze + + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex + CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex + + CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze + + included do + feature_category :package_registry + + helpers ::API::Helpers::PackagesManagerClientsHelpers + helpers ::API::Helpers::Packages::Conan::ApiHelpers + helpers ::API::Helpers::RelatedResourcesHelpers + + before do + require_packages_enabled! + + # Personal access token will be extracted from Bearer or Basic authorization + # in the overridden find_personal_access_token or find_user_from_job_token helpers + authenticate! + end + + desc 'Ping the Conan API' do + detail 'This feature was introduced in GitLab 12.2' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'ping' do + header 'X-Conan-Server-Capabilities', [].join(',') + end + + desc 'Search for packages' do + detail 'This feature was introduced in GitLab 12.4' + end + + params do + requires :q, type: String, desc: 'Search query' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'conans/search' do + service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute + service.payload + end + + namespace 'users' do + format :txt + + desc 'Authenticate user against conan CLI' do + detail 'This feature was introduced in GitLab 12.2' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'authenticate' do + unauthorized! unless token + + token.to_jwt + end + + desc 'Check for valid user credentials per conan CLI' do + detail 'This feature was introduced in GitLab 12.4' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'check_credentials' do + authenticate! + :ok + end + end + + params do + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' + requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + end + namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do + # Get the snapshot + # + # the snapshot is a hash of { filename: md5 hash } + # md5 hash is the has of that file. This hash is used to diff the files existing on the client + # to determine which client files need to be uploaded if no recipe exists the snapshot is empty + desc 'Package Snapshot' do + detail 'This feature was introduced in GitLab 12.5' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference' do + authorize!(:read_package, project) + + presenter = ::Packages::Conan::PackagePresenter.new( + package, + current_user, + project, + conan_package_reference: params[:conan_package_reference] + ) + + present presenter, with: ::API::Entities::ConanPackage::ConanPackageSnapshot + end + + desc 'Recipe Snapshot' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + authorize!(:read_package, project) + + presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) + + present presenter, with: ::API::Entities::ConanPackage::ConanRecipeSnapshot + end + + # Get the manifest + # returns the download urls for the existing recipe in the registry + # + # the manifest is a hash of { filename: url } + # where the url is the download url for the file + desc 'Package Digest' do + detail 'This feature was introduced in GitLab 12.5' + end + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference/digest' do + present_package_download_urls + end + + desc 'Recipe Digest' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'digest' do + present_recipe_download_urls + end + + # Get the download urls + # + # returns the download urls for the existing recipe or package in the registry + # + # the manifest is a hash of { filename: url } + # where the url is the download url for the file + desc 'Package Download Urls' do + detail 'This feature was introduced in GitLab 12.5' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference/download_urls' do + present_package_download_urls + end + + desc 'Recipe Download Urls' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'download_urls' do + present_recipe_download_urls + end + + # Get the upload urls + # + # request body contains { filename: filesize } where the filename is the + # name of the file the conan client is requesting to upload + # + # returns { filename: url } + # where the url is the upload url for the file that the conan client will use + desc 'Package Upload Urls' do + detail 'This feature was introduced in GitLab 12.4' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + post 'packages/:conan_package_reference/upload_urls' do + authorize!(:read_package, project) + + status 200 + present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls + end + + desc 'Recipe Upload Urls' do + detail 'This feature was introduced in GitLab 12.4' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + post 'upload_urls' do + authorize!(:read_package, project) + + status 200 + present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls + end + + desc 'Delete Package' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + delete do + authorize!(:destroy_package, project) + + track_package_event('delete_package', :conan, category: 'API::ConanPackages') + + package.destroy + end + end + + params do + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' + requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' + end + namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do + before do + authenticate_non_get! + end + + params do + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + end + namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do + desc 'Download recipe files' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + download_package_file(:recipe_file) + end + + desc 'Upload recipe package files' do + detail 'This feature was introduced in GitLab 12.6' + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put do + upload_package_file(:recipe_file) + end + + desc 'Workhorse authorize the conan recipe file' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) + end + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan Package ID' + requires :package_revision, type: String, desc: 'Conan Package Revision' + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + end + namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do + desc 'Download package files' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + download_package_file(:package_file) + end + + desc 'Workhorse authorize the conan package file' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) + end + + desc 'Upload package files' do + detail 'This feature was introduced in GitLab 12.6' + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put do + upload_package_file(:package_file) + end + end + end + end + end + end + end +end diff --git a/lib/api/entities/cluster.rb b/lib/api/entities/cluster.rb index 67459092a33..b7e76e763f7 100644 --- a/lib/api/entities/cluster.rb +++ b/lib/api/entities/cluster.rb @@ -3,7 +3,7 @@ module API module Entities class Cluster < Grape::Entity - expose :id, :name, :created_at, :domain + expose :id, :name, :created_at, :domain, :enabled, :managed expose :provider_type, :platform_type, :environment_scope, :cluster_type, :namespace_per_environment expose :user, using: Entities::UserBasic expose :platform_kubernetes, using: Entities::Platform::Kubernetes diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index a435b050042..81944a653c8 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -75,10 +75,12 @@ module API params do requires :cluster_id, type: Integer, desc: 'The cluster ID' optional :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, desc: 'Determines if cluster is active or not' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index cfb0c5fd705..6785b28ddef 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -83,6 +83,8 @@ module API optional :environment_scope, type: String, desc: 'The associated environment to the cluster' optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :enabled, type: Boolean, desc: 'Determines if cluster is active or not' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 65abee1f5eb..3faf07546de 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ayufan/openshift-cli +image: openshift/origin-cli stages: - build # dummy stage to follow the template guidelines diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3cbde9d30c8..5ea2363a0c5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: container_scanning: stage: test - image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + image: "$CS_ANALYZER_IMAGE" variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -18,6 +18,7 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: - name: $CLAIR_DB_IMAGE diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index e455bfac9de..910e711f046 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -56,5 +56,6 @@ cache: .destroy: &destroy stage: cleanup script: + - cd ${TF_ROOT} - gitlab-terraform destroy when: manual diff --git a/spec/requests/api/admin/instance_clusters_spec.rb b/spec/requests/api/admin/instance_clusters_spec.rb index 1052080aad4..ab3b6b718e1 100644 --- a/spec/requests/api/admin/instance_clusters_spec.rb +++ b/spec/requests/api/admin/instance_clusters_spec.rb @@ -90,6 +90,8 @@ RSpec.describe ::API::Admin::InstanceClusters do expect(json_response['environment_scope']).to eq('*') expect(json_response['cluster_type']).to eq('instance_type') expect(json_response['domain']).to eq('example.com') + expect(json_response['enabled']).to be_truthy + expect(json_response['managed']).to be_truthy end it 'returns kubernetes platform information' do @@ -163,6 +165,7 @@ RSpec.describe ::API::Admin::InstanceClusters do name: 'test-instance-cluster', domain: 'domain.example.com', managed: false, + enabled: false, namespace_per_environment: false, platform_kubernetes_attributes: platform_kubernetes_attributes, clusterable: clusterable @@ -205,9 +208,9 @@ RSpec.describe ::API::Admin::InstanceClusters do expect(cluster_result.name).to eq('test-instance-cluster') expect(cluster_result.domain).to eq('domain.example.com') expect(cluster_result.environment_scope).to eq('*') - expect(cluster_result.enabled).to eq(true) - expect(platform_kubernetes.authorization_type).to eq('rbac') expect(cluster_result.managed).to be_falsy + expect(cluster_result.enabled).to be_falsy + expect(platform_kubernetes.authorization_type).to eq('rbac') expect(cluster_result.namespace_per_environment).to eq(false) expect(platform_kubernetes.api_url).to eq("https://example.com") expect(platform_kubernetes.token).to eq('sample-token') @@ -301,6 +304,8 @@ RSpec.describe ::API::Admin::InstanceClusters do let(:update_params) do { domain: domain, + managed: false, + enabled: false, platform_kubernetes_attributes: platform_kubernetes_attributes } end @@ -326,6 +331,8 @@ RSpec.describe ::API::Admin::InstanceClusters do it 'updates cluster attributes' do expect(cluster.domain).to eq('new-domain.com') + expect(cluster.managed).to be_falsy + expect(cluster.enabled).to be_falsy end end @@ -338,6 +345,8 @@ RSpec.describe ::API::Admin::InstanceClusters do it 'does not update cluster attributes' do expect(cluster.domain).to eq('old-domain.com') + expect(cluster.managed).to be_truthy + expect(cluster.enabled).to be_truthy end it 'returns validation errors' do diff --git a/spec/requests/api/group_clusters_spec.rb b/spec/requests/api/group_clusters_spec.rb index eb21ae9468c..f65f9384efa 100644 --- a/spec/requests/api/group_clusters_spec.rb +++ b/spec/requests/api/group_clusters_spec.rb @@ -89,6 +89,8 @@ RSpec.describe API::GroupClusters do expect(json_response['environment_scope']).to eq('*') expect(json_response['cluster_type']).to eq('group_type') expect(json_response['domain']).to eq('example.com') + expect(json_response['enabled']).to be_truthy + expect(json_response['managed']).to be_truthy end it 'returns group information' do @@ -172,6 +174,7 @@ RSpec.describe API::GroupClusters do name: 'test-cluster', domain: 'domain.example.com', managed: false, + enabled: false, namespace_per_environment: false, platform_kubernetes_attributes: platform_kubernetes_attributes, management_project_id: management_project_id @@ -206,6 +209,7 @@ RSpec.describe API::GroupClusters do expect(cluster_result.name).to eq('test-cluster') expect(cluster_result.domain).to eq('domain.example.com') expect(cluster_result.managed).to be_falsy + expect(cluster_result.enabled).to be_falsy expect(cluster_result.management_project_id).to eq management_project_id expect(cluster_result.namespace_per_environment).to eq(false) expect(platform_kubernetes.rbac?).to be_truthy @@ -342,7 +346,9 @@ RSpec.describe API::GroupClusters do { domain: domain, platform_kubernetes_attributes: platform_kubernetes_attributes, - management_project_id: management_project_id + management_project_id: management_project_id, + managed: false, + enabled: false } end @@ -381,6 +387,8 @@ RSpec.describe API::GroupClusters do it 'updates cluster attributes' do expect(cluster.domain).to eq('new-domain.com') expect(cluster.management_project).to eq(management_project) + expect(cluster.managed).to be_falsy + expect(cluster.enabled).to be_falsy end end @@ -394,6 +402,8 @@ RSpec.describe API::GroupClusters do it 'does not update cluster attributes' do expect(cluster.domain).to eq('old-domain.com') expect(cluster.management_project).to be_nil + expect(cluster.managed).to be_truthy + expect(cluster.enabled).to be_truthy end it 'returns validation errors' do diff --git a/spec/requests/api/project_clusters_spec.rb b/spec/requests/api/project_clusters_spec.rb index 7b37862af74..f784f677c25 100644 --- a/spec/requests/api/project_clusters_spec.rb +++ b/spec/requests/api/project_clusters_spec.rb @@ -88,6 +88,8 @@ RSpec.describe API::ProjectClusters do expect(json_response['environment_scope']).to eq('*') expect(json_response['cluster_type']).to eq('project_type') expect(json_response['domain']).to eq('example.com') + expect(json_response['enabled']).to be_truthy + expect(json_response['managed']).to be_truthy end it 'returns project information' do @@ -171,6 +173,7 @@ RSpec.describe API::ProjectClusters do name: 'test-cluster', domain: 'domain.example.com', managed: false, + enabled: false, namespace_per_environment: false, platform_kubernetes_attributes: platform_kubernetes_attributes, management_project_id: management_project_id @@ -202,6 +205,7 @@ RSpec.describe API::ProjectClusters do expect(cluster_result.name).to eq('test-cluster') expect(cluster_result.domain).to eq('domain.example.com') expect(cluster_result.managed).to be_falsy + expect(cluster_result.enabled).to be_falsy expect(cluster_result.management_project_id).to eq management_project_id expect(cluster_result.namespace_per_environment).to eq(false) expect(platform_kubernetes.rbac?).to be_truthy @@ -337,7 +341,9 @@ RSpec.describe API::ProjectClusters do { domain: 'new-domain.com', platform_kubernetes_attributes: platform_kubernetes_attributes, - management_project_id: management_project_id + management_project_id: management_project_id, + managed: false, + enabled: false } end @@ -373,6 +379,8 @@ RSpec.describe API::ProjectClusters do it 'updates cluster attributes' do expect(response).to have_gitlab_http_status(:ok) expect(cluster.domain).to eq('new-domain.com') + expect(cluster.managed).to be_falsy + expect(cluster.enabled).to be_falsy expect(cluster.platform_kubernetes.namespace).to eq('new-namespace') expect(cluster.management_project).to eq(management_project) end @@ -384,6 +392,8 @@ RSpec.describe API::ProjectClusters do it 'does not update cluster attributes' do expect(response).to have_gitlab_http_status(:bad_request) expect(cluster.domain).not_to eq('new_domain.com') + expect(cluster.managed).to be_truthy + expect(cluster.enabled).to be_truthy expect(cluster.platform_kubernetes.namespace).not_to eq('invalid_namespace') expect(cluster.management_project).not_to eq(management_project) end |