diff options
72 files changed, 921 insertions, 702 deletions
diff --git a/.gitlab/issue_templates/QA Failure.md b/.gitlab/issue_templates/QA Failure.md index 9751b40cb91..957aac299b6 100644 --- a/.gitlab/issue_templates/QA Failure.md +++ b/.gitlab/issue_templates/QA Failure.md @@ -11,11 +11,16 @@ The issue should have the following: - A link to the failing job. - The stack trace from the job's logs in the "Stack trace" section below. - A screenshot (if available), and HTML capture (if available), in the "Screenshot / HTML page" section below. +- A link to the corresponding test case(s) in the summary. ---> ### Summary +Failing job(s): +Failing spec(s): + +Corresponding test case(s): ### Stack trace diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 6a2b312872d..3d1b5494241 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -c05b46905eacc7f4bd69f738fdf85ebb0b84ee4b +29e11b58a847bcf87055cea2f30afd7a9c2accb4 diff --git a/app/assets/javascripts/security_configuration/components/feature_card.vue b/app/assets/javascripts/security_configuration/components/feature_card.vue index 1c37d8008de..cd5ad86e1a8 100644 --- a/app/assets/javascripts/security_configuration/components/feature_card.vue +++ b/app/assets/javascripts/security_configuration/components/feature_card.vue @@ -31,13 +31,12 @@ export default { const button = this.enabled ? { text: this.$options.i18n.configureFeature, - category: 'secondary', } : { text: this.$options.i18n.enableFeature, - category: 'primary', }; + button.category = 'secondary'; button.text = sprintf(button.text, { feature: this.shortName }); return button; @@ -126,7 +125,7 @@ export default { v-else-if="showManageViaMr" :feature="feature" variant="confirm" - category="primary" + category="secondary" class="gl-mt-5" :data-qa-selector="`${feature.type}_mr_button`" @error="onError" diff --git a/app/graphql/queries/burndown_chart/burnup.query.graphql b/app/graphql/queries/burndown_chart/burnup.query.graphql index 7a389a6def5..0795645f8b7 100644 --- a/app/graphql/queries/burndown_chart/burnup.query.graphql +++ b/app/graphql/queries/burndown_chart/burnup.query.graphql @@ -1,4 +1,9 @@ -query BurnupTimesSeriesData($id: ID!, $isIteration: Boolean = false, $weight: Boolean = false) { +query BurnupTimesSeriesData( + $id: ID! + $isIteration: Boolean = false + $weight: Boolean = false + $fullPath: String +) { milestone(id: $id) @skip(if: $isIteration) { __typename id @@ -37,7 +42,7 @@ query BurnupTimesSeriesData($id: ID!, $isIteration: Boolean = false, $weight: Bo __typename id title - report { + report(fullPath: $fullPath) { __typename burnupTimeSeries { __typename diff --git a/app/graphql/types/dependency_proxy/image_ttl_group_policy_type.rb b/app/graphql/types/dependency_proxy/image_ttl_group_policy_type.rb index 29bba7122d0..9ab7c50998d 100644 --- a/app/graphql/types/dependency_proxy/image_ttl_group_policy_type.rb +++ b/app/graphql/types/dependency_proxy/image_ttl_group_policy_type.rb @@ -8,9 +8,9 @@ module Types authorize :read_dependency_proxy + field :created_at, Types::TimeType, null: true, description: 'Timestamp of creation.' field :enabled, GraphQL::Types::Boolean, null: false, description: 'Indicates whether the policy is enabled or disabled.' field :ttl, GraphQL::Types::Int, null: true, description: 'Number of days to retain a cached image file.' - field :created_at, Types::TimeType, null: true, description: 'Timestamp of creation.' field :updated_at, Types::TimeType, null: true, description: 'Timestamp of the most recent update.' end end diff --git a/app/graphql/types/dependency_proxy/manifest_type.rb b/app/graphql/types/dependency_proxy/manifest_type.rb index ef9f730df43..ab22f540f48 100644 --- a/app/graphql/types/dependency_proxy/manifest_type.rb +++ b/app/graphql/types/dependency_proxy/manifest_type.rb @@ -8,13 +8,13 @@ module Types authorize :read_dependency_proxy - field :id, ::Types::GlobalIDType[::DependencyProxy::Manifest], null: false, description: 'ID of the manifest.' field :created_at, Types::TimeType, null: false, description: 'Date of creation.' - field :updated_at, Types::TimeType, null: false, description: 'Date of most recent update.' + field :digest, GraphQL::Types::String, null: false, description: 'Digest of the manifest.' field :file_name, GraphQL::Types::String, null: false, description: 'Name of the manifest.' + field :id, ::Types::GlobalIDType[::DependencyProxy::Manifest], null: false, description: 'ID of the manifest.' field :image_name, GraphQL::Types::String, null: false, description: 'Name of the image.' field :size, GraphQL::Types::String, null: false, description: 'Size of the manifest file.' - field :digest, GraphQL::Types::String, null: false, description: 'Digest of the manifest.' + field :updated_at, Types::TimeType, null: false, description: 'Date of most recent update.' def image_name object.file_name.chomp(File.extname(object.file_name)) diff --git a/app/graphql/types/design_management/design_collection_type.rb b/app/graphql/types/design_management/design_collection_type.rb index 570eac907f3..91978aa37b0 100644 --- a/app/graphql/types/design_management/design_collection_type.rb +++ b/app/graphql/types/design_management/design_collection_type.rb @@ -8,10 +8,10 @@ module Types authorize :read_design - field :project, Types::ProjectType, null: false, - description: 'Project associated with the design collection.' field :issue, Types::IssueType, null: false, description: 'Issue associated with the design collection.' + field :project, Types::ProjectType, null: false, + description: 'Project associated with the design collection.' field :designs, Types::DesignManagement::DesignType.connection_type, diff --git a/app/graphql/types/diff_refs_type.rb b/app/graphql/types/diff_refs_type.rb index b19d09c789c..a03d72a4dc2 100644 --- a/app/graphql/types/diff_refs_type.rb +++ b/app/graphql/types/diff_refs_type.rb @@ -6,10 +6,10 @@ module Types class DiffRefsType < BaseObject graphql_name 'DiffRefs' - field :head_sha, GraphQL::Types::String, null: false, - description: 'SHA of the HEAD at the time the comment was made.' field :base_sha, GraphQL::Types::String, null: true, description: 'Merge base of the branch the comment was made on.' + field :head_sha, GraphQL::Types::String, null: false, + description: 'SHA of the HEAD at the time the comment was made.' field :start_sha, GraphQL::Types::String, null: false, description: 'SHA of the branch being compared against.' end diff --git a/app/graphql/types/diff_stats_summary_type.rb b/app/graphql/types/diff_stats_summary_type.rb index 079c73d0759..95705ddecf3 100644 --- a/app/graphql/types/diff_stats_summary_type.rb +++ b/app/graphql/types/diff_stats_summary_type.rb @@ -10,10 +10,10 @@ module Types field :additions, GraphQL::Types::Int, null: false, description: 'Number of lines added.' - field :deletions, GraphQL::Types::Int, null: false, - description: 'Number of lines deleted.' field :changes, GraphQL::Types::Int, null: false, description: 'Number of lines changed.' + field :deletions, GraphQL::Types::Int, null: false, + description: 'Number of lines deleted.' field :file_count, GraphQL::Types::Int, null: false, description: 'Number of files changed.' diff --git a/app/graphql/types/diff_stats_type.rb b/app/graphql/types/diff_stats_type.rb index 60aacca8ce5..da366fec8c3 100644 --- a/app/graphql/types/diff_stats_type.rb +++ b/app/graphql/types/diff_stats_type.rb @@ -8,12 +8,12 @@ module Types description 'Changes to a single file' - field :path, GraphQL::Types::String, null: false, - description: 'File path, relative to repository root.' field :additions, GraphQL::Types::Int, null: false, description: 'Number of lines added to this file.' field :deletions, GraphQL::Types::Int, null: false, description: 'Number of lines deleted from this file.' + field :path, GraphQL::Types::String, null: false, + description: 'File path, relative to repository root.' end # rubocop: enable Graphql/AuthorizeTypes end diff --git a/app/graphql/types/error_tracking/sentry_detailed_error_type.rb b/app/graphql/types/error_tracking/sentry_detailed_error_type.rb index 826ae61a1a3..b19ab80f96d 100644 --- a/app/graphql/types/error_tracking/sentry_detailed_error_type.rb +++ b/app/graphql/types/error_tracking/sentry_detailed_error_type.rb @@ -10,46 +10,68 @@ module Types authorize :read_sentry_issue - field :id, GraphQL::Types::ID, - null: false, - description: 'ID (global ID) of the error.' - field :integrated, GraphQL::Types::Boolean, - null: true, - description: 'Error tracking backend.' - field :sentry_id, GraphQL::Types::String, - method: :id, - null: false, - description: 'ID (Sentry ID) of the error.' - field :title, GraphQL::Types::String, + field :count, GraphQL::Types::Int, null: false, - description: 'Title of the error.' - field :type, GraphQL::Types::String, + description: 'Count of occurrences.' + field :culprit, GraphQL::Types::String, null: false, - description: 'Type of the error.' - field :user_count, GraphQL::Types::Int, + description: 'Culprit of the error.' + field :external_base_url, GraphQL::Types::String, null: false, - description: 'Count of users affected by the error.' - field :count, GraphQL::Types::Int, + description: 'External Base URL of the Sentry Instance.' + field :external_url, GraphQL::Types::String, null: false, - description: 'Count of occurrences.' + description: 'External URL of the error.' + field :first_release_last_commit, GraphQL::Types::String, + null: true, + description: 'Commit the error was first seen.' + field :first_release_short_version, GraphQL::Types::String, + null: true, + description: 'Release short version the error was first seen.' + field :first_release_version, GraphQL::Types::String, + null: true, + description: 'Release version the error was first seen.' field :first_seen, Types::TimeType, null: false, description: 'Timestamp when the error was first seen.' + field :frequency, [Types::ErrorTracking::SentryErrorFrequencyType], + null: false, + description: 'Last 24hr stats of the error.' + field :gitlab_commit, GraphQL::Types::String, + null: true, + description: 'GitLab commit SHA attributed to the Error based on the release version.' + field :gitlab_commit_path, GraphQL::Types::String, + null: true, + description: 'Path to the GitLab page for the GitLab commit attributed to the error.' + field :gitlab_issue_path, GraphQL::Types::String, + method: :gitlab_issue, + null: true, + description: 'URL of GitLab Issue.' + field :id, GraphQL::Types::ID, + null: false, + description: 'ID (global ID) of the error.' + field :integrated, GraphQL::Types::Boolean, + null: true, + description: 'Error tracking backend.' + field :last_release_last_commit, GraphQL::Types::String, + null: true, + description: 'Commit the error was last seen.' + field :last_release_short_version, GraphQL::Types::String, + null: true, + description: 'Release short version the error was last seen.' + field :last_release_version, GraphQL::Types::String, + null: true, + description: 'Release version the error was last seen.' field :last_seen, Types::TimeType, null: false, description: 'Timestamp when the error was last seen.' field :message, GraphQL::Types::String, null: true, description: 'Sentry metadata message of the error.' - field :culprit, GraphQL::Types::String, - null: false, - description: 'Culprit of the error.' - field :external_base_url, GraphQL::Types::String, - null: false, - description: 'External Base URL of the Sentry Instance.' - field :external_url, GraphQL::Types::String, + field :sentry_id, GraphQL::Types::String, + method: :id, null: false, - description: 'External URL of the error.' + description: 'ID (Sentry ID) of the error.' field :sentry_project_id, GraphQL::Types::ID, method: :project_id, null: false, @@ -68,40 +90,18 @@ module Types field :status, Types::ErrorTracking::SentryErrorStatusEnum, null: false, description: 'Status of the error.' - field :frequency, [Types::ErrorTracking::SentryErrorFrequencyType], - null: false, - description: 'Last 24hr stats of the error.' - field :first_release_last_commit, GraphQL::Types::String, - null: true, - description: 'Commit the error was first seen.' - field :last_release_last_commit, GraphQL::Types::String, - null: true, - description: 'Commit the error was last seen.' - field :first_release_short_version, GraphQL::Types::String, - null: true, - description: 'Release short version the error was first seen.' - field :last_release_short_version, GraphQL::Types::String, - null: true, - description: 'Release short version the error was last seen.' - field :first_release_version, GraphQL::Types::String, - null: true, - description: 'Release version the error was first seen.' - field :last_release_version, GraphQL::Types::String, - null: true, - description: 'Release version the error was last seen.' - field :gitlab_commit, GraphQL::Types::String, - null: true, - description: 'GitLab commit SHA attributed to the Error based on the release version.' - field :gitlab_commit_path, GraphQL::Types::String, - null: true, - description: 'Path to the GitLab page for the GitLab commit attributed to the error.' - field :gitlab_issue_path, GraphQL::Types::String, - method: :gitlab_issue, - null: true, - description: 'URL of GitLab Issue.' field :tags, Types::ErrorTracking::SentryErrorTagsType, null: false, description: 'Tags associated with the Sentry Error.' + field :title, GraphQL::Types::String, + null: false, + description: 'Title of the error.' + field :type, GraphQL::Types::String, + null: false, + description: 'Type of the error.' + field :user_count, GraphQL::Types::Int, + null: false, + description: 'Count of users affected by the error.' end end end diff --git a/app/graphql/types/error_tracking/sentry_error_collection_type.rb b/app/graphql/types/error_tracking/sentry_error_collection_type.rb index 2d8c3d3d326..9790560929b 100644 --- a/app/graphql/types/error_tracking/sentry_error_collection_type.rb +++ b/app/graphql/types/error_tracking/sentry_error_collection_type.rb @@ -8,15 +8,15 @@ module Types authorize :read_sentry_issue - field :errors, - description: "Collection of Sentry Errors.", - resolver: Resolvers::ErrorTracking::SentryErrorsResolver field :detailed_error, description: 'Detailed version of a Sentry error on the project.', resolver: Resolvers::ErrorTracking::SentryDetailedErrorResolver field :error_stack_trace, description: 'Stack Trace of Sentry Error.', resolver: Resolvers::ErrorTracking::SentryErrorStackTraceResolver + field :errors, + description: "Collection of Sentry Errors.", + resolver: Resolvers::ErrorTracking::SentryErrorsResolver field :external_url, GraphQL::Types::String, null: true, diff --git a/app/graphql/types/error_tracking/sentry_error_frequency_type.rb b/app/graphql/types/error_tracking/sentry_error_frequency_type.rb index 49a1b1e0476..f67becb3774 100644 --- a/app/graphql/types/error_tracking/sentry_error_frequency_type.rb +++ b/app/graphql/types/error_tracking/sentry_error_frequency_type.rb @@ -6,12 +6,12 @@ module Types class SentryErrorFrequencyType < ::Types::BaseObject graphql_name 'SentryErrorFrequency' - field :time, Types::TimeType, - null: false, - description: "Time the error frequency stats were recorded." field :count, GraphQL::Types::Int, null: false, description: "Count of errors received since the previously recorded time." + field :time, Types::TimeType, + null: false, + description: "Time the error frequency stats were recorded." end # rubocop: enable Graphql/AuthorizeTypes end diff --git a/app/graphql/types/error_tracking/sentry_error_stack_trace_context_type.rb b/app/graphql/types/error_tracking/sentry_error_stack_trace_context_type.rb index ad31854b30c..d4b806c4e1e 100644 --- a/app/graphql/types/error_tracking/sentry_error_stack_trace_context_type.rb +++ b/app/graphql/types/error_tracking/sentry_error_stack_trace_context_type.rb @@ -7,14 +7,14 @@ module Types graphql_name 'SentryErrorStackTraceContext' description 'An object context for a Sentry error stack trace' - field :line, - GraphQL::Types::Int, - null: false, - description: 'Line number of the context.' field :code, GraphQL::Types::String, null: false, description: 'Code number of the context.' + field :line, + GraphQL::Types::Int, + null: false, + description: 'Line number of the context.' def line object[0] diff --git a/app/graphql/types/error_tracking/sentry_error_stack_trace_entry_type.rb b/app/graphql/types/error_tracking/sentry_error_stack_trace_entry_type.rb index e8f78004569..c33baa06052 100644 --- a/app/graphql/types/error_tracking/sentry_error_stack_trace_entry_type.rb +++ b/app/graphql/types/error_tracking/sentry_error_stack_trace_entry_type.rb @@ -7,18 +7,18 @@ module Types graphql_name 'SentryErrorStackTraceEntry' description 'An object containing a stack trace entry for a Sentry error' - field :function, GraphQL::Types::String, + field :col, GraphQL::Types::String, null: true, description: 'Function in which the Sentry error occurred.' - field :col, GraphQL::Types::String, + field :file_name, GraphQL::Types::String, + null: true, + description: 'File in which the Sentry error occurred.' + field :function, GraphQL::Types::String, null: true, description: 'Function in which the Sentry error occurred.' field :line, GraphQL::Types::String, null: true, description: 'Function in which the Sentry error occurred.' - field :file_name, GraphQL::Types::String, - null: true, - description: 'File in which the Sentry error occurred.' field :trace_context, [Types::ErrorTracking::SentryErrorStackTraceContextType], null: true, description: 'Context of the Sentry error.' diff --git a/app/models/preloaders/environments/deployment_preloader.rb b/app/models/preloaders/environments/deployment_preloader.rb index fcf892698bb..592d3e54a30 100644 --- a/app/models/preloaders/environments/deployment_preloader.rb +++ b/app/models/preloaders/environments/deployment_preloader.rb @@ -34,8 +34,16 @@ module Preloaders deployments_by_environment_id = deployments.index_by(&:environment_id) environments.each do |environment| - environment.association(association_name).target = deployments_by_environment_id[environment.id] + associated_deployment = deployments_by_environment_id[environment.id] + + environment.association(association_name).target = associated_deployment environment.association(association_name).loaded! + + if associated_deployment + # `last?` in DeploymentEntity requires this environment to be loaded + associated_deployment.association(:environment).target = environment + associated_deployment.association(:environment).loaded! + end end end end diff --git a/app/models/release.rb b/app/models/release.rb index 0fda6940249..7f3af50e91e 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -5,6 +5,7 @@ class Release < ApplicationRecord include CacheMarkdownField include Importable include Gitlab::Utils::StrongMemoize + include EachBatch cache_markdown_field :description diff --git a/app/models/user.rb b/app/models/user.rb index 74832bff9ac..5322fe1142a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -2166,7 +2166,7 @@ class User < ApplicationRecord end def signup_email_invalid_message - self.new_record? ? _('is not allowed for sign-up.') : _('is not allowed.') + self.new_record? ? _('is not allowed for sign-up. Please use your regular email address.') : _('is not allowed. Please use your regular email address.') end def check_username_format diff --git a/config/feature_flags/development/snippets_binary_blob.yml b/config/feature_flags/development/snippets_binary_blob.yml deleted file mode 100644 index 72a959858d9..00000000000 --- a/config/feature_flags/development/snippets_binary_blob.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: snippets_binary_blob -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37419 -rollout_issue_url: -milestone: '13.3' -type: development -group: group::editor -default_enabled: false diff --git a/config/feature_flags/development/usage_data_i_snippets_show.yml b/config/feature_flags/development/usage_data_i_snippets_show.yml deleted file mode 100644 index 446338d5b95..00000000000 --- a/config/feature_flags/development/usage_data_i_snippets_show.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: usage_data_i_snippets_show -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48113 -rollout_issue_url: -milestone: '13.7' -type: development -group: group::editor -default_enabled: true diff --git a/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb b/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb new file mode 100644 index 00000000000..2bb5ba18743 --- /dev/null +++ b/db/post_migrate/20220222192524_create_not_null_constraint_releases_tag.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateNotNullConstraintReleasesTag < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + add_not_null_constraint :releases, :tag, constraint_name: 'releases_not_null_tag', validate: false + end + + def down + remove_not_null_constraint :releases, :tag, constraint_name: 'releases_not_null_tag' + end +end diff --git a/db/post_migrate/20220222192525_remove_null_releases.rb b/db/post_migrate/20220222192525_remove_null_releases.rb new file mode 100644 index 00000000000..4efd5393122 --- /dev/null +++ b/db/post_migrate/20220222192525_remove_null_releases.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RemoveNullReleases < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + class Release < ActiveRecord::Base + include EachBatch + + self.table_name = 'releases' + end + + def up + Release.all.each_batch(of: 25000) do |rel| + rel.where(tag: nil).delete_all + end + end + + def down + # no-op + # + # releases with the same tag within a project have been removed + # and therefore the duplicate release data is no longer available + end +end diff --git a/db/schema_migrations/20220222192524 b/db/schema_migrations/20220222192524 new file mode 100644 index 00000000000..c49e45f0d61 --- /dev/null +++ b/db/schema_migrations/20220222192524 @@ -0,0 +1 @@ +b876119bb369a9831736cddf5326b72a74003ec2e17fe863654cb69497fcf236
\ No newline at end of file diff --git a/db/schema_migrations/20220222192525 b/db/schema_migrations/20220222192525 new file mode 100644 index 00000000000..6eeec13bbb5 --- /dev/null +++ b/db/schema_migrations/20220222192525 @@ -0,0 +1 @@ +f512ea4c4a2625c647c3d05765152fee963b56962b674f839180fd77c194ccb0
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 613fe897401..ab87ddf896c 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25064,6 +25064,9 @@ ALTER TABLE ONLY related_epic_links ALTER TABLE ONLY release_links ADD CONSTRAINT release_links_pkey PRIMARY KEY (id); +ALTER TABLE releases + ADD CONSTRAINT releases_not_null_tag CHECK ((tag IS NOT NULL)) NOT VALID; + ALTER TABLE ONLY releases ADD CONSTRAINT releases_pkey PRIMARY KEY (id); diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md index 7d365f73101..7dd2c3b3b29 100644 --- a/doc/administration/geo/setup/index.md +++ b/doc/administration/geo/setup/index.md @@ -10,7 +10,7 @@ type: howto These instructions assume you have a working instance of GitLab. They guide you through: 1. Making your existing instance the **primary** site. -1. Adding **secondary** site(s). +1. Adding **secondary** sites. WARNING: The steps below should be followed in the order they appear. **Make sure the GitLab version is the same on all sites.** @@ -19,15 +19,15 @@ The steps below should be followed in the order they appear. **Make sure the Git If you installed GitLab using the Omnibus packages (highly recommended): -1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the node(s) that will serve as the **secondary** site. Do not create an account or log in to the new **secondary** site. +1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the nodes that will serve as the **secondary** site. Do not create an account or log in to the new **secondary** site. 1. [Upload the GitLab License](../../../user/admin_area/license.md) on the **primary** site to unlock Geo. The license must be for [GitLab Premium](https://about.gitlab.com/pricing/) or higher. 1. [Set up the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology). -1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** site(s). -1. [Configure GitLab](../replication/configuration.md) to set the **primary** and **secondary** site(s). -1. Optional: [Configure a secondary LDAP server](../../auth/ldap/index.md) for the **secondary** site(s). See [notes on LDAP](../index.md#ldap). +1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** sites. +1. [Configure GitLab](../replication/configuration.md) to set the **primary** and **secondary** sites. +1. Optional: [Configure a secondary LDAP server](../../auth/ldap/index.md) for the **secondary** sites. See [notes on LDAP](../index.md#ldap). 1. Follow the [Using a Geo Site](../replication/usage.md) guide. 1. [Configure Geo secondary proxying](../secondary_proxy/index.md) to use a single, unified URL for all Geo sites. This step is recommended to accelerate most read requests while transparently proxying writes to the primary Geo site. ## Post-installation documentation -After installing GitLab on the **secondary** site(s) and performing the initial configuration, see the [following documentation for post-installation information](../index.md#post-installation-documentation). +After installing GitLab on the **secondary** sites and performing the initial configuration, see the [following documentation for post-installation information](../index.md#post-installation-documentation). diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 420f2191ef2..c303dc1fd19 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -62,7 +62,7 @@ Using the consolidated object storage configuration has a number of advantages: - It enables the use of [encrypted S3 buckets](#encrypted-s3-buckets). - It [uploads files to S3 with proper `Content-MD5` headers](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). -Because [direct upload mode](../development/uploads.md#direct-upload) +Because [direct upload mode](../development/uploads/implementation.md#direct-upload) must be enabled, only the following providers can be used: - [Amazon S3-compatible providers](#s3-compatible-connection-settings) diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index d1113125c4e..5e8f7cb18bb 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -141,10 +141,10 @@ You can optionally track progress and verify that all packages migrated successf - `sudo gitlab-rails dbconsole` for Omnibus GitLab instances. - `sudo -u git -H psql -d gitlabhq_production` for source-installed instances. -Verify `objectstg` below (where `store=2`) has count of all states: +Verify `objectstg` below (where `file_store=2`) has count of all states: ```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when store = '1' then 1 else 0 end) AS filesystem, sum(case when store = '2' then 1 else 0 end) AS objectstg FROM terraform_states; +gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM terraform_states; total | filesystem | objectstg ------+------------+----------- diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 55c3f85bfb9..aba7b5fc318 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -67,7 +67,7 @@ For source installations the following settings are nested under `uploads:` and |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Uploads will be stored| | -| `direct_upload` | Set to `true` to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads.md#direct-upload). | `false` | +| `direct_upload` | Set to `true` to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads/implementation.md#direct-upload). | `false` | | `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 (if `direct_upload` is set to `true` it will override `background_upload`) | `true` | | `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 29f9bd3e2ee..87ff6549540 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -281,7 +281,8 @@ Deployments created by users on GitLab Premium or higher include the `approvals` "avatar_url": "https://www.gravatar.com/avatar/e83ac685f68ea07553ad3054c738c709?s=80&d=identicon", "web_url": "http://localhost:3000/project_6_bot" }, - "status": "approved" + "status": "approved", + "created_at": "2022-02-24T20:22:30.097Z" } ], ... @@ -351,7 +352,8 @@ Deployments created by users on GitLab Premium or higher include the `approvals` "avatar_url": "https://www.gravatar.com/avatar/e83ac685f68ea07553ad3054c738c709?s=80&d=identicon", "web_url": "http://localhost:3000/project_6_bot" }, - "status": "approved" + "status": "approved", + "created_at": "2022-02-24T20:22:30.097Z" } ], ... @@ -417,7 +419,8 @@ Deployments created by users on GitLab Premium or higher include the `approvals` "avatar_url": "https://www.gravatar.com/avatar/e83ac685f68ea07553ad3054c738c709?s=80&d=identicon", "web_url": "http://localhost:3000/project_6_bot" }, - "status": "approved" + "status": "approved", + "created_at": "2022-02-24T20:22:30.097Z" } ], ... @@ -480,6 +483,7 @@ Example response: "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", "web_url": "http://localhost:3000/root" }, - "status": "approved" + "status": "approved", + "created_at": "2022-02-24T20:22:30.097Z" } ``` diff --git a/doc/architecture/blueprints/object_storage/index.md b/doc/architecture/blueprints/object_storage/index.md index 7864c951eca..3420caa325f 100644 --- a/doc/architecture/blueprints/object_storage/index.md +++ b/doc/architecture/blueprints/object_storage/index.md @@ -196,7 +196,7 @@ require one bucket. ## Additional reading materials -- [Uploads development documentation: The problem description](../../../development/uploads.md#the-problem-description). +- [Uploads development guide](../../../development/uploads/index.md). - [Speed up the monolith, building a smart reverse proxy in Go](https://archive.fosdem.org/2020/schedule/event/speedupmonolith/): a presentation explaining a bit of workhorse history and the challenge we faced in releasing the first cloud-native installation. - [Object Storage improvements epic](https://gitlab.com/groups/gitlab-org/-/epics/483). - We are moving to GraphQL API, but [we do not support direct upload](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 3664ca7642a..57663efc9f9 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -624,7 +624,7 @@ Enterprise Edition instance. This has some implications: [added to Omnibus](https://docs.gitlab.com/omnibus/settings/gitlab.yml#adding-a-new-setting-to-gitlabyml). 1. **File system access** is not possible in a [cloud-native architecture](architecture.md#adapting-existing-and-introducing-new-components). Ensure that we support object storage for any file storage we need to perform. For more - information, see the [uploads documentation](uploads.md). + information, see the [uploads documentation](uploads/index.md). ### Review turnaround time diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 6798d02f046..a9b4d13ab06 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -81,7 +81,7 @@ request is as follows: 1. If your MR touches code that executes shell commands, reads or opens files, or handles paths to files on disk, make sure it adheres to the [shell command guidelines](../shell_commands.md) -1. If your code needs to handle file storage, see the [uploads documentation](../uploads.md). +1. If your code needs to handle file storage, see the [uploads documentation](../uploads/index.md). 1. If your merge request adds one or more migrations, make sure to execute all migrations on a fresh database before the MR is reviewed. If the review leads to large changes in the MR, execute the migrations again once the review is complete. diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index 139825655e9..2e64f52651e 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -47,7 +47,7 @@ The Content Editor requires two properties: - `renderMarkdown` is an asynchronous function that returns the response (String) of invoking the [Markdown API](../../api/markdown.md). -- `uploadsPath` is a URL that points to a [GitLab upload service](../uploads.md#upload-encodings) +- `uploadsPath` is a URL that points to a [GitLab upload service](../uploads/implementation.md#upload-encodings) with `multipart/form-data` support. See the [`WikiForm.vue`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/pages/shared/wikis/components/wiki_form.vue#L207) diff --git a/doc/development/file_storage.md b/doc/development/file_storage.md index d161206f44d..04e2f381c97 100644 --- a/doc/development/file_storage.md +++ b/doc/development/file_storage.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w We use the [CarrierWave](https://github.com/carrierwaveuploader/carrierwave) gem to handle file upload, store and retrieval. -File uploads should be accelerated by workhorse, for details please refer to [uploads development documentation](uploads.md). +File uploads should be accelerated by workhorse, for details please refer to [uploads development documentation](uploads/index.md). There are many places where file uploading is used, according to contexts: diff --git a/doc/development/index.md b/doc/development/index.md index 0501f70b818..552555ab85c 100644 --- a/doc/development/index.md +++ b/doc/development/index.md @@ -227,7 +227,7 @@ the [reviewer values](https://about.gitlab.com/handbook/engineering/workflow/rev - [Working with merge request diffs](diffs.md) - [Approval Rules](approval_rules.md) - [Repository mirroring](repository_mirroring.md) -- [File uploads](uploads.md) +- [Uploads development guide](uploads/index.md) - [Auto DevOps development guide](auto_devops.md) - [Renaming features](renaming_features.md) - [Code Intelligence](code_intelligence/index.md) diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index 5fe138e9cd2..40f02f4fb6f 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -526,7 +526,7 @@ end The usage of shared temporary storage is required if your intent is to persistent file for a disk-based storage, and not Object Storage. -[Workhorse direct_upload](uploads.md#direct-upload) when accepting file +[Workhorse direct_upload](uploads/implementation.md#direct-upload) when accepting file can write it to shared storage, and later GitLab Rails can perform a move operation. The move operation on the same destination is instantaneous. The system instead of performing `copy` operation just re-attaches file into a new place. @@ -550,7 +550,7 @@ that implements a seamless support for Shared and Object Storage-based persisten #### Data access Each feature that accepts data uploads or allows to download them needs to use -[Workhorse direct_upload](uploads.md#direct-upload). It means that uploads needs to be +[Workhorse direct_upload](uploads/implementation.md#direct-upload). It means that uploads needs to be saved directly to Object Storage by Workhorse, and all downloads needs to be served by Workhorse. @@ -562,5 +562,5 @@ can time out, which is especially problematic for slow clients. If clients take to upload/download the processing slot might be killed due to request processing timeout (usually between 30s-60s). -For the above reasons it is required that [Workhorse direct_upload](uploads.md#direct-upload) is implemented +For the above reasons it is required that [Workhorse direct_upload](uploads/implementation.md#direct-upload) is implemented for all file uploads and downloads. diff --git a/doc/development/packages.md b/doc/development/packages.md index 38c1b941eaf..35a93c77c7f 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -151,7 +151,7 @@ During this phase, the idea is to collect as much information as possible about 1. Empty file structure (API file, base service for this package) 1. Authentication system for "logging in" to the package manager 1. Identify metadata and create applicable tables - 1. Workhorse route for [object storage direct upload](uploads.md#direct-upload) + 1. Workhorse route for [object storage direct upload](uploads/implementation.md#direct-upload) 1. Endpoints required for upload/publish 1. Endpoints required for install/download 1. Endpoints required for required actions @@ -210,7 +210,7 @@ File uploads should be handled by GitLab Workhorse using object accelerated uplo the workhorse proxy that checks all incoming requests to GitLab intercept the upload request, upload the file, and forward a request to the main GitLab codebase only containing the metadata and file location rather than the file itself. An overview of this process can be found in the -[development documentation](uploads.md#direct-upload). +[development documentation](uploads/implementation.md#direct-upload). In terms of code, this means a route must be added to the [GitLab Workhorse project](https://gitlab.com/gitlab-org/gitlab-workhorse) for each upload endpoint being added @@ -272,7 +272,7 @@ features must be implemented when the feature flag is removed. - File format guards (only accept valid file formats for the package type) - Name regex with validation - Version regex with validation -- Workhorse route for [accelerated](uploads.md#how-to-add-a-new-upload-route) uploads +- Workhorse route for [accelerated](uploads/working_with_uploads.md) uploads - Background workers for extracting package metadata (if applicable) - Documentation (how to use the feature) - API Documentation (individual endpoints with curl examples) diff --git a/doc/development/shared_files.md b/doc/development/shared_files.md index e26464a5d80..4f13bb80761 100644 --- a/doc/development/shared_files.md +++ b/doc/development/shared_files.md @@ -11,5 +11,5 @@ servers in `shared/`, using a shared storage solution like NFS. Although this is some GitLab installations, it must not be the only file storage option for a given feature. This is because [cloud-native GitLab installations do not support it](architecture.md#adapting-existing-and-introducing-new-components). -Our [uploads documentation](uploads.md) describes how to handle file storage in +Our [uploads documentation](uploads/index.md) describes how to handle file storage in such a way that it supports both options: direct disk access and object storage. diff --git a/doc/development/uploads.md b/doc/development/uploads.md index 14ecf38e21c..1860f898a26 100644 --- a/doc/development/uploads.md +++ b/doc/development/uploads.md @@ -1,430 +1,9 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'uploads/index.md' +remove_date: '2022-04-30' --- -# Uploads development documentation +This document was moved to [another location](uploads/index.md). -[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) has special rules for handling uploads. -We process the upload in Workhorse to prevent occupying a Ruby process on I/O operations and because it is cheaper. -This process can also directly upload to object storage. - -## The problem description - -The following graph explains machine boundaries in a scalable GitLab installation. Without any Workhorse optimization in place, we can expect incoming requests to follow the numbers on the arrows. - -```mermaid -graph TB - subgraph "load balancers" - LB(Proxy) - end - - subgraph "Shared storage" - nfs(NFS) - end - - subgraph "redis cluster" - r(persisted redis) - end - LB-- 1 -->Workhorse - - subgraph "web or API fleet" - Workhorse-- 2 -->rails - end - rails-- "3 (write files)" -->nfs - rails-- "4 (schedule a job)" -->r - - subgraph sidekiq - s(sidekiq) - end - s-- "5 (fetch a job)" -->r - s-- "6 (read files)" -->nfs -``` - -We have three challenges here: performance, availability, and scalability. - -### Performance - -Rails process are expensive in terms of both CPU and memory. Ruby [global interpreter lock](https://en.wikipedia.org/wiki/Global_interpreter_lock) adds to cost too because the Ruby process spends time on I/O operations on step 3 causing incoming requests to pile up. - -In order to improve this, [disk buffered upload](#disk-buffered-upload) was implemented. With this, Rails no longer deals with writing uploaded files to disk. - -```mermaid -graph TB - subgraph "load balancers" - LB(HA Proxy) - end - - subgraph "Shared storage" - nfs(NFS) - end - - subgraph "redis cluster" - r(persisted redis) - end - LB-- 1 -->Workhorse - - subgraph "web or API fleet" - Workhorse-- "3 (without files)" -->rails - end - Workhorse -- "2 (write files)" -->nfs - rails-- "4 (schedule a job)" -->r - - subgraph sidekiq - s(sidekiq) - end - s-- "5 (fetch a job)" -->r - s-- "6 (read files)" -->nfs -``` - -### Availability - -There's also an availability problem in this setup, NFS is a [single point of failure](https://en.wikipedia.org/wiki/Single_point_of_failure). - -To address this problem an HA object storage can be used and it's supported by [direct upload](#direct-upload) - -### Scalability - -Scaling NFS is outside of our support scope, and NFS is not a part of cloud native installations. - -All features that require Sidekiq and do not use direct upload doesn't work without NFS. In Kubernetes, machine boundaries translate to PODs, and in this case the uploaded file is written into the POD private disk. Since Sidekiq POD cannot reach into other pods, the operation fails to read it. - -## How to select the proper level of acceleration? - -Selecting the proper acceleration is a tradeoff between speed of development and operational costs. - -We can identify three major use-cases for an upload: - -1. **storage:** if we are uploading for storing a file (like artifacts, packages, or discussion attachments). In this case [direct upload](#direct-upload) is the proper level as it's the less resource-intensive operation. Additional information can be found on [File Storage in GitLab](file_storage.md). -1. **in-controller/synchronous processing:** if we allow processing **small files** synchronously, using [disk buffered upload](#disk-buffered-upload) may speed up development. -1. **Sidekiq/asynchronous processing:** Asynchronous processing must implement [direct upload](#direct-upload), the reason being that it's the only way to support Cloud Native deployments without a shared NFS. - -For more details about currently broken feature see [epic &1802](https://gitlab.com/groups/gitlab-org/-/epics/1802). - -### Handling repository uploads - -Some features involves Git repository uploads without using a regular Git client. -Some examples are uploading a repository file from the web interface and [design management](../user/project/issues/design_management.md). - -Those uploads requires the rails controller to act as a Git client in lieu of the user. -Those operation falls into _in-controller/synchronous processing_ category, but we have no warranties on the file size. - -In case of a LFS upload, the file pointer is committed synchronously, but file upload to object storage is performed asynchronously with Sidekiq. - -## Upload encodings - -By upload encoding we mean how the file is included within the incoming request. - -We have three kinds of file encoding in our uploads: - -1. <i class="fa fa-check-circle"></i> **multipart**: `multipart/form-data` is the most common, a file is encoded as a part of a multipart encoded request. -1. <i class="fa fa-check-circle"></i> **body**: some APIs uploads files as the whole request body. -1. <i class="fa fa-times-circle"></i> **JSON**: some JSON APIs upload files as base64-encoded strings. This requires a change to GitLab Workhorse, - which is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/325068). - -## Uploading technologies - -By uploading technologies we mean how all the involved services interact with each other. - -GitLab supports 3 kinds of uploading technologies, here follows a brief description with a sequence diagram for each one. Diagrams are not meant to be exhaustive. - -### Rack Multipart upload - -This is the default kind of upload, and it's the most expensive in terms of resources. - -In this case, Workhorse is unaware of files being uploaded and acts as a regular proxy. - -When a multipart request reaches the rails application, `Rack::Multipart` leaves behind temporary files in `/tmp` and uses valuable Ruby process time to copy files around. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - - activate c - c ->>+w: POST /some/url/upload - w->>+r: POST /some/url/upload - - r->>r: save the incoming file on /tmp - r->>r: read the file for processing - - r-->>-c: request result - deactivate c - deactivate w -``` - -### Disk buffered upload - -This kind of upload avoids wasting resources caused by handling upload writes to `/tmp` in rails. - -This optimization is not active by default on REST API requests. - -When enabled, Workhorse looks for files in multipart MIME requests, uploading -any it finds to a temporary file on shared storage. The MIME data in the request -is replaced with the path to the corresponding file before it is forwarded to -Rails. - -To prevent abuse of this feature, Workhorse signs the modified request with a -special header, stating which entries it modified. Rails ignores any -unsigned path entries. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - participant s as NFS - - activate c - c ->>+w: POST /some/url/upload - - w->>+s: save the incoming file on a temporary location - s-->>-w: request result - - w->>+r: POST /some/url/upload - Note over w,r: file was replaced with its location<br>and other metadata - - opt requires async processing - r->>+redis: schedule a job - redis-->>-r: job is scheduled - end - - r-->>-c: request result - deactivate c - w->>-w: cleanup - - opt requires async processing - activate sidekiq - sidekiq->>+redis: fetch a job - redis-->>-sidekiq: job - - sidekiq->>+s: read file - s-->>-sidekiq: file - - sidekiq->>sidekiq: process file - - deactivate sidekiq - end -``` - -### Direct upload - -This is the more advanced acceleration technique we have in place. - -Workhorse asks Rails for temporary pre-signed object storage URLs and directly uploads to object storage. - -In this setup, an extra Rails route must be implemented in order to handle authorization. Examples of this can be found in: - -- [`Projects::LfsStorageController`](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/app/controllers/projects/lfs_storage_controller.rb) - and [its routes](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/config/routes/git_http.rb#L31-32). -- [API endpoints for uploading packages](packages.md#file-uploads). - -Direct upload falls back to _disk buffered upload_ when `direct_upload` is disabled inside the [object storage setting](../administration/uploads.md#object-storage-settings). -The answer to the `/authorize` call contains only a file system path. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - participant os as Object Storage - - activate c - c ->>+w: POST /some/url/upload - - w ->>+r: POST /some/url/upload/authorize - Note over w,r: this request has an empty body - r-->>-w: presigned OS URL - - w->>+os: PUT file - Note over w,os: file is stored on a temporary location. Rails select the destination - os-->>-w: request result - - w->>+r: POST /some/url/upload - Note over w,r: file was replaced with its location<br>and other metadata - - r->>+os: move object to final destination - os-->>-r: request result - - opt requires async processing - r->>+redis: schedule a job - redis-->>-r: job is scheduled - end - - r-->>-c: request result - deactivate c - w->>-w: cleanup - - opt requires async processing - activate sidekiq - sidekiq->>+redis: fetch a job - redis-->>-sidekiq: job - - sidekiq->>+os: get object - os-->>-sidekiq: file - - sidekiq->>sidekiq: process file - - deactivate sidekiq - end -``` - -## How to add a new upload route - -In this section, we describe how to add a new upload route [accelerated](#uploading-technologies) by Workhorse for [body and multipart](#upload-encodings) encoded uploads. - -Upload routes belong to one of these categories: - -1. Rails controllers: uploads handled by Rails controllers. -1. Grape API: uploads handled by a Grape API endpoint. -1. GraphQL API: uploads handled by a GraphQL resolve function. - -WARNING: -GraphQL uploads do not support [direct upload](#direct-upload) yet. Depending on the use case, the feature may not work on installations without NFS (like GitLab.com or Kubernetes installations). Uploading to object storage inside the GraphQL resolve function may result in timeout errors. For more details please follow [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). - -### Update Workhorse for the new route - -For both the Rails controller and Grape API uploads, Workhorse has to be updated in order to get the -support for the new upload route. - -1. Open a new issue in the [Workhorse tracker](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/new) describing precisely the new upload route: - - The route's URL. - - The [upload encoding](#upload-encodings). - - If possible, provide a dump of the upload request. -1. Implement and get the MR merged for this issue above. -1. Ask the Maintainers of [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to create a new release. You can do that in the MR - directly during the maintainer review or ask for it in the `#workhorse` Slack channel. -1. Bump the [Workhorse version file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) - to the version you have from the previous points, or bump it in the same merge request that contains - the Rails changes (see [Implementing the new route with a Rails controller](#implementing-the-new-route-with-a-rails-controller) or [Implementing the new route with a Grape API endpoint](#implementing-the-new-route-with-a-grape-api-endpoint) below). - -### Implementing the new route with a Rails controller - -For a Rails controller upload, we usually have a [multipart](#upload-encodings) upload and there are a -few things to do: - -1. The upload is available under the parameter name you're using. For example, it could be an `artifact` - or a nested parameter such as `user[avatar]`. Let's say that we have the upload under the - `file` parameter, reading `params[:file]` should get you an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance. -1. Generally speaking, it's a good idea to check if the instance is from the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) class. For example, see how we checked -[that the parameter is indeed an `UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/commit/ea30fe8a71bf16ba07f1050ab4820607b5658719#51c0cc7a17b7f12c32bc41cfab3649ff2739b0eb_79_77). - -WARNING: -**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -instance using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` -passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -instance that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) -builds automatically for you. - -### Implementing the new route with a Grape API endpoint - -For a Grape API upload, we can have [body or a multipart](#upload-encodings) upload. Things are slightly more complicated: two endpoints are needed. One for the -Workhorse pre-upload authorization and one for accepting the upload metadata from Workhorse: - -1. Implement an endpoint with the URL + `/authorize` suffix that will: - - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the [API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). - - Check user permissions. - - Set the status to `200` with `status 200`. - - Set the content type with `content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE`. - - Use your dedicated `Uploader` class (let's say that it's `FileUploader`) to build the response with `FileUploader.workhorse_authorize(params)`. -1. Implement the endpoint for the upload request that will: - - Require all the `UploadedFile` objects as parameters. - - For example, if we expect a single parameter `file` to be an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance, -use `requires :file, type: ::API::Validations::Types::WorkhorseFile`. - - Body upload requests have their upload available under the parameter `file`. - - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the -[API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). - - Check the user permissions. - - The remaining code of the processing. This is where the code must be reading the parameter (for -our example, it would be `params[:file]`). - -WARNING: -**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -object using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` -passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -object that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) -builds automatically for you. - -### Document Object Storage buckets and CarrierWave integration - -When using Object Storage, GitLab expects each kind of upload to maintain its own bucket in the respective -Object Storage destination. Moreover, the integration with CarrierWave is not used all the time. -The [Object Storage Working Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/) -is investigating an approach that unifies Object Storage buckets into a single one and removes CarrierWave -so as to simplify implementation and administration of uploads. - -Therefore, document new uploads here by slotting them into the following tables: - -- [Feature bucket details](#feature-bucket-details) -- [CarrierWave integration](#carrierwave-integration) - -#### Feature bucket details - -| Feature | Upload technology | Uploader | Bucket structure | -|------------------------------------------|-------------------|-----------------------|-----------------------------------------------------------------------------------------------------------| -| Job artifacts | `direct upload` | `workhorse` | `/artifacts/<proj_id_hash>/<date>/<job_id>/<artifact_id>` | -| Pipeline artifacts | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/pipelines/<pipeline_id>/artifacts/<artifact_id>` | -| Live job traces | `fog` | `sidekiq` | `/artifacts/tmp/builds/<job_id>/chunks/<chunk_index>.log` | -| Job traces archive | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/<date>/<job_id>/<artifact_id>/job.log` | -| Autoscale runner caching | N/A | `gitlab-runner` | `/gitlab-com-[platform-]runners-cache/???` | -| Backups | N/A | `s3cmd`, `awscli`, or `gcs` | `/gitlab-backups/???` | -| Git LFS | `direct upload` | `workhorse` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | -| Design management files | `disk buffering` | `rails controller` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | -| Design management thumbnails | `carrierwave` | `sidekiq` | `/uploads/design_management/action/image_v432x230/<model_id>` | -| Generic file uploads | `direct upload` | `workhorse` | `/uploads/@hashed/[0:2]/[2:4]/<hash1>/<hash2>/file` | -| Generic file uploads - personal snippets | `direct upload` | `workhorse` | `/uploads/personal_snippet/<snippet_id>/<filename>` | -| Global appearance settings | `disk buffering` | `rails controller` | `/uploads/appearance/...` | -| Topics | `disk buffering` | `rails controller` | `/uploads/projects/topic/...` | -| Avatar images | `direct upload` | `workhorse` | `/uploads/[user,group,project]/avatar/<model_id>` | -| Import/export | `direct upload` | `workhorse` | `/uploads/import_export_upload/???` | -| GitLab Migration | `carrierwave` | `sidekiq` | `/uploads/bulk_imports/???` | -| MR diffs | `carrierwave` | `sidekiq` | `/external-diffs/merge_request_diffs/mr-<mr_id>/diff-<diff_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<proj_id_hash>/packages/<pkg_segment>/files/<pkg_file_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_component_file/<component_file_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_distribution/<distribution_file_id>` | -| Container image cache (?) | `direct upload` | `workhorse` | `/dependency-proxy/<group_id_hash>/dependency_proxy/<group_id>/files/<proxy_id>/<blob_id or manifest_id>` | -| Terraform state files | `carrierwave` | `rails controller` | `/terraform/<proj_id_hash>/<terraform_state_id>` | -| Pages content archives | `carrierwave` | `sidekiq` | `/gitlab-gprd-pages/<proj_id_hash>/pages_deployments/<deployment_id>/` | -| Secure Files | `carrierwave` | `sidekiq` | `/ci-secure-files/<proj_id_hash>/secure_files/<secure_file_id>/` | - -#### CarrierWave integration - -| File | Carrierwave usage | Categorized | -|---------------------------------------------------------|----------------------------------------------------------------------------------|---------------------| -| `app/models/project.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/projects/topic.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/group.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/user.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/terraform/state_version.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/ci/job_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/ci/pipeline_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/pages_deployment.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/lfs_object.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/dependency_proxy/blob.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/dependency_proxy/manifest.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/packages/composer/cache_file.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/packages/package_file.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/concerns/packages/debian/component_file.rb` | `include FileStoreMounter` | :white_check_mark: | -| `ee/app/models/issuable_metric_image.rb` | `include FileStoreMounter` | | -| `ee/app/models/vulnerabilities/remediation.rb` | `include FileStoreMounter` | | -| `ee/app/models/vulnerabilities/export.rb` | `include FileStoreMounter` | | -| `app/models/packages/debian/project_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | -| `app/models/packages/debian/group_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | -| `app/models/packages/debian/project_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | -| `app/models/packages/debian/group_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | -| `app/models/merge_request_diff.rb` | `mount_uploader :external_diff, ExternalDiffUploader` | :white_check_mark: | -| `app/models/note.rb` | `mount_uploader :attachment, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :logo, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :header_logo, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :favicon, FaviconUploader` | :white_check_mark: | -| `app/models/project.rb` | `mount_uploader :bfg_object_map, AttachmentUploader` | | -| `app/models/import_export_upload.rb` | `mount_uploader :import_file, ImportExportUploader` | :white_check_mark: | -| `app/models/import_export_upload.rb` | `mount_uploader :export_file, ImportExportUploader` | :white_check_mark: | -| `app/models/ci/deleted_object.rb` | `mount_uploader :file, DeletedObjectUploader` | | -| `app/models/design_management/action.rb` | `mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader` | :white_check_mark: | -| `app/models/concerns/packages/debian/distribution.rb` | `mount_uploader :signed_file, Packages::Debian::DistributionReleaseFileUploader` | :white_check_mark: | -| `app/models/bulk_imports/export_upload.rb` | `mount_uploader :export_file, ExportUploader` | :white_check_mark: | -| `ee/app/models/user_permission_export_upload.rb` | `mount_uploader :file, AttachmentUploader` | | -| `app/models/ci/secure_file.rb` | `include FileStoreMounter` | | +<!-- This redirect file can be deleted after 2022-04-30. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/uploads/background.md b/doc/development/uploads/background.md new file mode 100644 index 00000000000..bff659881f4 --- /dev/null +++ b/doc/development/uploads/background.md @@ -0,0 +1,94 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Uploads guide: Why GitLab uses custom upload logic + +This page is for developers trying to better understand the history behind GitLab uploads and the +technical challenges associated with uploads. + +## The problem description + +[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) has special rules for handling uploads. +We process the upload in Workhorse to prevent occupying a Ruby process on I/O operations and because it is cheaper. +This process can also directly upload to object storage. + +The following graph explains machine boundaries in a scalable GitLab installation. Without any Workhorse optimization in place, we can expect incoming requests to follow the numbers on the arrows. + +```mermaid +graph TB + subgraph "load balancers" + LB(Proxy) + end + + subgraph "Shared storage" + nfs(NFS) + end + + subgraph "redis cluster" + r(persisted redis) + end + LB-- 1 -->Workhorse + + subgraph "web or API fleet" + Workhorse-- 2 -->rails + end + rails-- "3 (write files)" -->nfs + rails-- "4 (schedule a job)" -->r + + subgraph sidekiq + s(sidekiq) + end + s-- "5 (fetch a job)" -->r + s-- "6 (read files)" -->nfs +``` + +We have three challenges here: performance, availability, and scalability. + +### Performance + +Rails process are expensive in terms of both CPU and memory. Ruby [global interpreter lock](https://en.wikipedia.org/wiki/Global_interpreter_lock) adds to cost too because the Ruby process spends time on I/O operations on step 3 causing incoming requests to pile up. + +In order to improve this, [disk buffered upload](implementation.md#disk-buffered-upload) was implemented. With this, Rails no longer deals with writing uploaded files to disk. + +```mermaid +graph TB + subgraph "load balancers" + LB(HA Proxy) + end + + subgraph "Shared storage" + nfs(NFS) + end + + subgraph "redis cluster" + r(persisted redis) + end + LB-- 1 -->Workhorse + + subgraph "web or API fleet" + Workhorse-- "3 (without files)" -->rails + end + Workhorse -- "2 (write files)" -->nfs + rails-- "4 (schedule a job)" -->r + + subgraph sidekiq + s(sidekiq) + end + s-- "5 (fetch a job)" -->r + s-- "6 (read files)" -->nfs +``` + +### Availability + +There's also an availability problem in this setup, NFS is a [single point of failure](https://en.wikipedia.org/wiki/Single_point_of_failure). + +To address this problem an HA object storage can be used and it's supported by [direct upload](implementation.md#direct-upload) + +### Scalability + +Scaling NFS is outside of our support scope, and NFS is not a part of cloud native installations. + +All features that require Sidekiq and do not use direct upload doesn't work without NFS. In Kubernetes, machine boundaries translate to PODs, and in this case the uploaded file is written into the POD private disk. Since Sidekiq POD cannot reach into other pods, the operation fails to read it. diff --git a/doc/development/uploads/implementation.md b/doc/development/uploads/implementation.md new file mode 100644 index 00000000000..13a875cd1af --- /dev/null +++ b/doc/development/uploads/implementation.md @@ -0,0 +1,190 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Uploads guide: How uploads work technically + +This page is for developers trying to better understand what kinds of uploads exist in GitLab and how they are implemented. + +## Kinds of uploads and how to choose between them + +We can identify three major use-cases for an upload: + +1. **storage:** if we are uploading for storing a file (like artifacts, packages, or discussion attachments). In this case [direct upload](#direct-upload) is the proper level as it's the less resource-intensive operation. Additional information can be found on [File Storage in GitLab](../file_storage.md). +1. **in-controller/synchronous processing:** if we allow processing **small files** synchronously, using [disk buffered upload](#disk-buffered-upload) may speed up development. +1. **Sidekiq/asynchronous processing:** Asynchronous processing must implement [direct upload](#direct-upload), the reason being that it's the only way to support Cloud Native deployments without a shared NFS. + +Selecting the proper acceleration is a tradeoff between speed of development and operational costs. + +For more details about currently broken feature see [epic &1802](https://gitlab.com/groups/gitlab-org/-/epics/1802). + +### Handling repository uploads + +Some features involves Git repository uploads without using a regular Git client. +Some examples are uploading a repository file from the web interface and [design management](../../user/project/issues/design_management.md). + +Those uploads requires the rails controller to act as a Git client in lieu of the user. +Those operation falls into _in-controller/synchronous processing_ category, but we have no warranties on the file size. + +In case of a LFS upload, the file pointer is committed synchronously, but file upload to object storage is performed asynchronously with Sidekiq. + +## Upload encodings + +By upload encoding we mean how the file is included within the incoming request. + +We have three kinds of file encoding in our uploads: + +1. <i class="fa fa-check-circle"></i> **multipart**: `multipart/form-data` is the most common, a file is encoded as a part of a multipart encoded request. +1. <i class="fa fa-check-circle"></i> **body**: some APIs uploads files as the whole request body. +1. <i class="fa fa-times-circle"></i> **JSON**: some JSON APIs upload files as base64-encoded strings. This requires a change to GitLab Workhorse, + which is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/325068). + +## Uploading technologies + +By uploading technologies we mean how all the involved services interact with each other. + +GitLab supports 3 kinds of uploading technologies, here follows a brief description with a sequence diagram for each one. Diagrams are not meant to be exhaustive. + +### Rack Multipart upload + +This is the default kind of upload, and it's the most expensive in terms of resources. + +In this case, Workhorse is unaware of files being uploaded and acts as a regular proxy. + +When a multipart request reaches the rails application, `Rack::Multipart` leaves behind temporary files in `/tmp` and uses valuable Ruby process time to copy files around. + +```mermaid +sequenceDiagram + participant c as Client + participant w as Workhorse + participant r as Rails + + activate c + c ->>+w: POST /some/url/upload + w->>+r: POST /some/url/upload + + r->>r: save the incoming file on /tmp + r->>r: read the file for processing + + r-->>-c: request result + deactivate c + deactivate w +``` + +### Disk buffered upload + +This kind of upload avoids wasting resources caused by handling upload writes to `/tmp` in rails. + +This optimization is not active by default on REST API requests. + +When enabled, Workhorse looks for files in multipart MIME requests, uploading +any it finds to a temporary file on shared storage. The MIME data in the request +is replaced with the path to the corresponding file before it is forwarded to +Rails. + +To prevent abuse of this feature, Workhorse signs the modified request with a +special header, stating which entries it modified. Rails ignores any +unsigned path entries. + +```mermaid +sequenceDiagram + participant c as Client + participant w as Workhorse + participant r as Rails + participant s as NFS + + activate c + c ->>+w: POST /some/url/upload + + w->>+s: save the incoming file on a temporary location + s-->>-w: request result + + w->>+r: POST /some/url/upload + Note over w,r: file was replaced with its location<br>and other metadata + + opt requires async processing + r->>+redis: schedule a job + redis-->>-r: job is scheduled + end + + r-->>-c: request result + deactivate c + w->>-w: cleanup + + opt requires async processing + activate sidekiq + sidekiq->>+redis: fetch a job + redis-->>-sidekiq: job + + sidekiq->>+s: read file + s-->>-sidekiq: file + + sidekiq->>sidekiq: process file + + deactivate sidekiq + end +``` + +### Direct upload + +This is the more advanced acceleration technique we have in place. + +Workhorse asks Rails for temporary pre-signed object storage URLs and directly uploads to object storage. + +In this setup, an extra Rails route must be implemented in order to handle authorization. Examples of this can be found in: + +- [`Projects::LfsStorageController`](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/app/controllers/projects/lfs_storage_controller.rb) + and [its routes](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/config/routes/git_http.rb#L31-32). +- [API endpoints for uploading packages](../packages.md#file-uploads). + +Direct upload falls back to _disk buffered upload_ when `direct_upload` is disabled inside the [object storage setting](../../administration/uploads.md#object-storage-settings). +The answer to the `/authorize` call contains only a file system path. + +```mermaid +sequenceDiagram + participant c as Client + participant w as Workhorse + participant r as Rails + participant os as Object Storage + + activate c + c ->>+w: POST /some/url/upload + + w ->>+r: POST /some/url/upload/authorize + Note over w,r: this request has an empty body + r-->>-w: presigned OS URL + + w->>+os: PUT file + Note over w,os: file is stored on a temporary location. Rails select the destination + os-->>-w: request result + + w->>+r: POST /some/url/upload + Note over w,r: file was replaced with its location<br>and other metadata + + r->>+os: move object to final destination + os-->>-r: request result + + opt requires async processing + r->>+redis: schedule a job + redis-->>-r: job is scheduled + end + + r-->>-c: request result + deactivate c + w->>-w: cleanup + + opt requires async processing + activate sidekiq + sidekiq->>+redis: fetch a job + redis-->>-sidekiq: job + + sidekiq->>+os: get object + os-->>-sidekiq: file + + sidekiq->>sidekiq: process file + + deactivate sidekiq + end +``` diff --git a/doc/development/uploads/index.md b/doc/development/uploads/index.md new file mode 100644 index 00000000000..c486f2d3689 --- /dev/null +++ b/doc/development/uploads/index.md @@ -0,0 +1,14 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Uploads development guide + +Uploads are an integral part of many GitLab features. To understand how GitLab handles uploads, refer to +the following pages: + +- [Why GitLab uses custom upload logic.](background.md) +- [How uploads work technically.](implementation.md) +- [How to add new uploads.](working_with_uploads.md) diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md new file mode 100644 index 00000000000..99c04888804 --- /dev/null +++ b/doc/development/uploads/working_with_uploads.md @@ -0,0 +1,163 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Uploads guide: Adding new uploads + +In this section, we describe how to add a new upload route [accelerated](implementation.md#uploading-technologies) by Workhorse for [body and multipart](implementation.md#upload-encodings) encoded uploads. + +Upload routes belong to one of these categories: + +1. Rails controllers: uploads handled by Rails controllers. +1. Grape API: uploads handled by a Grape API endpoint. +1. GraphQL API: uploads handled by a GraphQL resolve function. + +WARNING: +GraphQL uploads do not support [direct upload](implementation.md#direct-upload) yet. Depending on the use case, the feature may not work on installations without NFS (like GitLab.com or Kubernetes installations). Uploading to object storage inside the GraphQL resolve function may result in timeout errors. For more details please follow [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). + +## Update Workhorse for the new route + +For both the Rails controller and Grape API uploads, Workhorse has to be updated in order to get the +support for the new upload route. + +1. Open a new issue in the [Workhorse tracker](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/new) describing precisely the new upload route: + - The route's URL. + - The [upload encoding](implementation.md#upload-encodings). + - If possible, provide a dump of the upload request. +1. Implement and get the MR merged for this issue above. +1. Ask the Maintainers of [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to create a new release. You can do that in the MR + directly during the maintainer review or ask for it in the `#workhorse` Slack channel. +1. Bump the [Workhorse version file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) + to the version you have from the previous points, or bump it in the same merge request that contains + the Rails changes (see [Implementing the new route with a Rails controller](#implementing-the-new-route-with-a-rails-controller) or [Implementing the new route with a Grape API endpoint](#implementing-the-new-route-with-a-grape-api-endpoint) below). + +## Implementing the new route with a Rails controller + +For a Rails controller upload, we usually have a [multipart](implementation.md#upload-encodings) upload and there are a +few things to do: + +1. The upload is available under the parameter name you're using. For example, it could be an `artifact` + or a nested parameter such as `user[avatar]`. Let's say that we have the upload under the + `file` parameter, reading `params[:file]` should get you an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance. +1. Generally speaking, it's a good idea to check if the instance is from the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) class. For example, see how we checked +[that the parameter is indeed an `UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/commit/ea30fe8a71bf16ba07f1050ab4820607b5658719#51c0cc7a17b7f12c32bc41cfab3649ff2739b0eb_79_77). + +WARNING: +**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) +instance using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` +passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) +instance that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) +builds automatically for you. + +## Implementing the new route with a Grape API endpoint + +For a Grape API upload, we can have [body or a multipart](implementation.md#upload-encodings) upload. Things are slightly more complicated: two endpoints are needed. One for the +Workhorse pre-upload authorization and one for accepting the upload metadata from Workhorse: + +1. Implement an endpoint with the URL + `/authorize` suffix that will: + - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the [API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). + - Check user permissions. + - Set the status to `200` with `status 200`. + - Set the content type with `content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE`. + - Use your dedicated `Uploader` class (let's say that it's `FileUploader`) to build the response with `FileUploader.workhorse_authorize(params)`. +1. Implement the endpoint for the upload request that will: + - Require all the `UploadedFile` objects as parameters. + - For example, if we expect a single parameter `file` to be an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance, +use `requires :file, type: ::API::Validations::Types::WorkhorseFile`. + - Body upload requests have their upload available under the parameter `file`. + - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the +[API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). + - Check the user permissions. + - The remaining code of the processing. This is where the code must be reading the parameter (for +our example, it would be `params[:file]`). + +WARNING: +**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) +object using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` +passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) +object that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) +builds automatically for you. + +## Document Object Storage buckets and CarrierWave integration + +When using Object Storage, GitLab expects each kind of upload to maintain its own bucket in the respective +Object Storage destination. Moreover, the integration with CarrierWave is not used all the time. +The [Object Storage Working Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/) +is investigating an approach that unifies Object Storage buckets into a single one and removes CarrierWave +so as to simplify implementation and administration of uploads. + +Therefore, document new uploads here by slotting them into the following tables: + +- [Feature bucket details](#feature-bucket-details) +- [CarrierWave integration](#carrierwave-integration) + +### Feature bucket details + +| Feature | Upload technology | Uploader | Bucket structure | +|------------------------------------------|-------------------|-----------------------|-----------------------------------------------------------------------------------------------------------| +| Job artifacts | `direct upload` | `workhorse` | `/artifacts/<proj_id_hash>/<date>/<job_id>/<artifact_id>` | +| Pipeline artifacts | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/pipelines/<pipeline_id>/artifacts/<artifact_id>` | +| Live job traces | `fog` | `sidekiq` | `/artifacts/tmp/builds/<job_id>/chunks/<chunk_index>.log` | +| Job traces archive | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/<date>/<job_id>/<artifact_id>/job.log` | +| Autoscale runner caching | N/A | `gitlab-runner` | `/gitlab-com-[platform-]runners-cache/???` | +| Backups | N/A | `s3cmd`, `awscli`, or `gcs` | `/gitlab-backups/???` | +| Git LFS | `direct upload` | `workhorse` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | +| Design management files | `disk buffering` | `rails controller` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | +| Design management thumbnails | `carrierwave` | `sidekiq` | `/uploads/design_management/action/image_v432x230/<model_id>` | +| Generic file uploads | `direct upload` | `workhorse` | `/uploads/@hashed/[0:2]/[2:4]/<hash1>/<hash2>/file` | +| Generic file uploads - personal snippets | `direct upload` | `workhorse` | `/uploads/personal_snippet/<snippet_id>/<filename>` | +| Global appearance settings | `disk buffering` | `rails controller` | `/uploads/appearance/...` | +| Topics | `disk buffering` | `rails controller` | `/uploads/projects/topic/...` | +| Avatar images | `direct upload` | `workhorse` | `/uploads/[user,group,project]/avatar/<model_id>` | +| Import/export | `direct upload` | `workhorse` | `/uploads/import_export_upload/???` | +| GitLab Migration | `carrierwave` | `sidekiq` | `/uploads/bulk_imports/???` | +| MR diffs | `carrierwave` | `sidekiq` | `/external-diffs/merge_request_diffs/mr-<mr_id>/diff-<diff_id>` | +| Package manager archives | `direct upload` | `sidekiq` | `/packages/<proj_id_hash>/packages/<pkg_segment>/files/<pkg_file_id>` | +| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_component_file/<component_file_id>` | +| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_distribution/<distribution_file_id>` | +| Container image cache (?) | `direct upload` | `workhorse` | `/dependency-proxy/<group_id_hash>/dependency_proxy/<group_id>/files/<proxy_id>/<blob_id or manifest_id>` | +| Terraform state files | `carrierwave` | `rails controller` | `/terraform/<proj_id_hash>/<terraform_state_id>` | +| Pages content archives | `carrierwave` | `sidekiq` | `/gitlab-gprd-pages/<proj_id_hash>/pages_deployments/<deployment_id>/` | +| Secure Files | `carrierwave` | `sidekiq` | `/ci-secure-files/<proj_id_hash>/secure_files/<secure_file_id>/` | + +### CarrierWave integration + +| File | Carrierwave usage | Categorized | +|---------------------------------------------------------|----------------------------------------------------------------------------------|---------------------| +| `app/models/project.rb` | `include Avatarable` | :white_check_mark: | +| `app/models/projects/topic.rb` | `include Avatarable` | :white_check_mark: | +| `app/models/group.rb` | `include Avatarable` | :white_check_mark: | +| `app/models/user.rb` | `include Avatarable` | :white_check_mark: | +| `app/models/terraform/state_version.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/ci/job_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/ci/pipeline_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/pages_deployment.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/lfs_object.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/dependency_proxy/blob.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/dependency_proxy/manifest.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/packages/composer/cache_file.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/packages/package_file.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/concerns/packages/debian/component_file.rb` | `include FileStoreMounter` | :white_check_mark: | +| `ee/app/models/issuable_metric_image.rb` | `include FileStoreMounter` | | +| `ee/app/models/vulnerabilities/remediation.rb` | `include FileStoreMounter` | | +| `ee/app/models/vulnerabilities/export.rb` | `include FileStoreMounter` | | +| `app/models/packages/debian/project_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | +| `app/models/packages/debian/group_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | +| `app/models/packages/debian/project_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | +| `app/models/packages/debian/group_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | +| `app/models/merge_request_diff.rb` | `mount_uploader :external_diff, ExternalDiffUploader` | :white_check_mark: | +| `app/models/note.rb` | `mount_uploader :attachment, AttachmentUploader` | :white_check_mark: | +| `app/models/appearance.rb` | `mount_uploader :logo, AttachmentUploader` | :white_check_mark: | +| `app/models/appearance.rb` | `mount_uploader :header_logo, AttachmentUploader` | :white_check_mark: | +| `app/models/appearance.rb` | `mount_uploader :favicon, FaviconUploader` | :white_check_mark: | +| `app/models/project.rb` | `mount_uploader :bfg_object_map, AttachmentUploader` | | +| `app/models/import_export_upload.rb` | `mount_uploader :import_file, ImportExportUploader` | :white_check_mark: | +| `app/models/import_export_upload.rb` | `mount_uploader :export_file, ImportExportUploader` | :white_check_mark: | +| `app/models/ci/deleted_object.rb` | `mount_uploader :file, DeletedObjectUploader` | | +| `app/models/design_management/action.rb` | `mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader` | :white_check_mark: | +| `app/models/concerns/packages/debian/distribution.rb` | `mount_uploader :signed_file, Packages::Debian::DistributionReleaseFileUploader` | :white_check_mark: | +| `app/models/bulk_imports/export_upload.rb` | `mount_uploader :export_file, ExportUploader` | :white_check_mark: | +| `ee/app/models/user_permission_export_upload.rb` | `mount_uploader :file, AttachmentUploader` | | +| `app/models/ci/secure_file.rb` | `include FileStoreMounter` | | diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index 430f8e1a2a2..cf37abc8d47 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -49,7 +49,9 @@ You can configure the following security controls: - Select **Configure with a merge request** to create a merge request with the changes required to enable Dependency Scanning. For more details, see [Enable Dependency Scanning via an automatic merge request](../dependency_scanning/index.md#enable-dependency-scanning-via-an-automatic-merge-request). - [Container Scanning](../container_scanning/index.md) - - Can be configured with `.gitlab-ci.yml`. For more details, read [Container Scanning](../../../user/application_security/container_scanning/index.md#configuration). + - Select **Configure with a merge request** to create a merge request with the changes required to + enable Container Scanning. For more details, see + [Enable Container Scanning through an automatic merge request](../container_scanning/index.md#enable-container-scanning-through-an-automatic-merge-request). - [Cluster Image Scanning](../cluster_image_scanning/index.md) - Can be configured with `.gitlab-ci.yml`. For more details, read [Cluster Image Scanning](../../../user/application_security/cluster_image_scanning/#configuration). - [Secret Detection](../secret_detection/index.md) diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 453174cd8d7..7dcbaf9e649 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -253,6 +253,24 @@ images. To configure the images, set the `CS_ANALYZER_IMAGE` variable to the sta | Grype | `registry.gitlab.com/security-products/container-scanning/grype:4-ubi` | | Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:4-ubi` | +### Enable Container Scanning through an automatic merge request + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6334) in GitLab 14.9. + +To enable Container Scanning in a project, create a merge request from the Security Configuration +page: + +1. In the project where you want to enable Container Scanning, go to + **Security & Compliance > Configuration**. +1. In the **Container Scanning** row, select **Configure with a merge request**. + +This automatically creates a merge request with the changes necessary to enable Container Scanning. +To complete the configuration, review and merge this merge request. + +The configuration tool works best with no existing `.gitlab-ci.yml` file, or with a minimal +configuration file. If you have a complex GitLab configuration file, it may not be parsed +successfully and an error may occur. + ### Overriding the container scanning template If you want to override the job definition (for example, to change properties like `variables`), you diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index 6f2c616f48a..8e6449fe639 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -52,66 +52,44 @@ To update a Kubernetes cluster by using GitOps, complete the following steps. Any time you commit updates to your Kubernetes manifests, the agent updates the cluster. -### GitOps configuration reference +## GitOps configuration reference -The following snippet shows the possible keys and values for the GitOps section of an agent configuration file. +The following snippet shows an example of the possible keys and values for the GitOps section of an agent configuration file. ```yaml gitops: - # The agent watches projects where your manifests are stored. - # When a project changes, the agent deploys the changes to the cluster. manifest_projects: - # No authentication mechanisms are currently supported. - # The `id` is a path to a Git repository that has - # Kubernetes manifests in YAML or JSON format. - id: gitlab-org/cluster-integration/gitlab-agent - # Namespace to use if not set explicitly in object manifest. - # Also used for inventory ConfigMap objects. default_namespace: my-ns - # Paths in the repository you want to scan for manifest files. - # Directories with names that start with a dot are ignored. paths: - # Read all .yaml files from team1/app1 directory. - # See https://github.com/bmatcuk/doublestar#about and - # https://pkg.go.dev/github.com/bmatcuk/doublestar/v2#Match for globbing rules. + # Read all YAML files from this directory. - glob: '/team1/app1/*.yaml' # Read all .yaml files from team2/apps and all subdirectories. - glob: '/team2/apps/**/*.yaml' # If 'paths' is not specified or is an empty list, the configuration below is used. - glob: '/**/*.{yaml,yml,json}' - # Reconcile timeout defines whether the applier should wait - # until all applied resources have been reconciled, and if so, - # how long to wait. - reconcile_timeout: 3600s # 1 hour by default - # Dry run strategy defines whether changes should actually be performed, - # or if it is just talk and no action. - # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/common/common.go#L68-L89 - # Can be: none, client, server - dry_run_strategy: none # 'none' by default - # Prune defines whether pruning of previously applied - # objects should happen after apply. - prune: true # enabled by default - # Prune timeout defines whether we should wait for all resources - # to be fully deleted after pruning, and if so, how long we should - # wait. - prune_timeout: 3600s # 1 hour by default - # Prune propagation policy defines the deletion propagation policy - # that should be used for pruning. - # https://github.com/kubernetes/apimachinery/blob/44113beed5d39f1b261a12ec398a356e02358307/pkg/apis/meta/v1/types.go#L456-L470 - # Can be: orphan, background, foreground - prune_propagation_policy: foreground # 'foreground' by default - # Inventory policy defines if an inventory object can take over - # objects that belong to another inventory object or don't - # belong to any inventory object. - # This is done by determining if the apply/prune operation - # can go through for a resource based on comparison of - # the inventory-id value in the package and the owning-inventory - # annotation (config.k8s.io/owning-inventory) in the live object. - # https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/inventory/policy.go#L12-L66 - # Can be: must_match, adopt_if_no_inventory, adopt_all - inventory_policy: must_match # 'must_match' by default + reconcile_timeout: 3600s + dry_run_strategy: none + prune: true + prune_timeout: 3600s + prune_propagation_policy: foreground + inventory_policy: must_match ``` +| Keyword | Description | +|--|--| +| `manifest_projects` | Projects where your Kubernetes manifests are stored. The agent monitors the files in the repositories in these projects. When manifest files change, the agent deploys the changes to the cluster. | +| `id` | Required. Path to a Git repository that has Kubernetes manifests in YAML or JSON format. No authentication mechanisms are currently supported. | +| `default_namespace` | Namespace to use if not set explicitly in object manifest. Also used for inventory `ConfigMap` objects. | +| `paths` | Repository paths to scan for manifest files. Directories with names that start with a dot `(.)` are ignored. | +| `paths[].glob` | Required. See [doublestar](https://github.com/bmatcuk/doublestar#about) and [the match function](https://pkg.go.dev/github.com/bmatcuk/doublestar/v2#Match) for globbing rules. | +| `reconcile_timeout` | Determines whether the applier should wait until all applied resources have been reconciled, and if so, how long to wait. Default is 3600 seconds (1 hour). | +| `dry_run_strategy` | Determines whether changes [should be performed](https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/common/common.go#L68-L89). Can be: `none`, `client`, or `server`. Default is `none`.| +| `prune` | Determines whether pruning of previously applied objects should happen after apply. Default is `true`. | +| `prune_timeout` | Determines whether to wait for all resources to be fully deleted after pruning, and if so, how long to wait. Default is 3600 seconds (1 hour). | +| `prune_propagation_policy` | The deletion propagation policy that [should be used for pruning](https://github.com/kubernetes/apimachinery/blob/44113beed5d39f1b261a12ec398a356e02358307/pkg/apis/meta/v1/types.go#L456-L470). Can be: `orphan`, `background`, or `foreground`. Default is `foreground`. | +| `inventory_policy` | Determines whether an inventory object can take over objects that belong to another inventory object or don't belong to any inventory object. This is done by determining if the apply/prune operation can go through for a resource based on comparison of the `inventory-id` value in the package and the `owning-inventory` annotation (`config.k8s.io/owning-inventory`) [in the live object](https://github.com/kubernetes-sigs/cli-utils/blob/d6968048dcd80b1c7b55d9e4f31fc25f71c9b490/pkg/inventory/policy.go#L12-L66). Can be: `must_match`, `adopt_if_no_inventory`, or `adopt_all`. Default is `must_match`. | + ## Additional resources The following documentation and examples can help you get started with a GitOps workflow. diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index c0bd6f1a672..b5912a0b40e 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -142,6 +142,7 @@ To view an iteration report, go to the iterations list page and select an iterat > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222750) in GitLab 13.6. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/269972) in GitLab 13.7. +> - Scoped burnup and burndown charts in subgroups and projects [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326029) in GitLab 14.9. The iteration report includes [burndown and burnup charts](../../project/milestones/burndown_and_burnup_charts.md), similar to how they appear when viewing a [milestone](../../project/milestones/index.md). @@ -149,6 +150,33 @@ similar to how they appear when viewing a [milestone](../../project/milestones/i Burndown charts help track completion progress of total scope, and burnup charts track the daily total count and weight of issues added to and completed in a given timebox. +#### Iteration charts scoped to subgroups or projects + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326029) in GitLab 14.9. + +You can view burndown and burnup charts for iterations created for a group in any of its +subgroups or projects. +When you do this, the charts only count the issues that belong to the subgroup or project. + +For example, suppose a group has two projects named `Project 1` and `Project 2`. +Each project has a single issue assigned to the same iteration from the group. + +An iteration report generated for the group shows issue counts for all the group's projects: + +- Completed: 0 of 2 +- Incomplete: 0 of 2 +- Unstarted: 2 of 2 +- Burndown chart total issues: 2 +- Burnup chart total issues: 2 + +An iteration report generated for `Project 1` shows only issues that belong to this project: + +- Completed: 0 of 1 +- Incomplete: 0 of 1 +- Unstarted: 1 of 1 +- Burndown chart total issues: 1 +- Burnup chart total issues: 1 + ### Group issues by label > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225500) in GitLab 13.8. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 71440298d85..428cb4a7f17 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -53,7 +53,7 @@ the issue board feature. > - Multiple issue boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53811) to GitLab Free in 12.1. > - Multiple issue boards per group are available in GitLab Premium. -Multiple issue boards allow for more than one issue board for a given project **(FREE)** or group **(PREMIUM)**. +Multiple issue boards allow for more than one issue board for a given project in GitLab Free or group in GitLab Premium and higher tiers. This is great for large projects with more than one team or when a repository hosts the code of multiple products. Using the search box at the top of the menu, you can filter the listed boards. @@ -275,7 +275,7 @@ Users on GitLab Free can use a single group issue board. ### Assignee lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5784) in GitLab 11.0. As in a regular list showing all issues with a chosen label, you can add an assignee list that shows all issues assigned to a user. @@ -295,7 +295,7 @@ To remove an assignee list, just as with a label list, click the trash icon. ### Milestone lists **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6469) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6469) in GitLab 11.2. You're also able to create lists of a milestone. These are lists that filter issues by the assigned milestone, giving you more freedom and visibility on the issue board. To add a milestone list: @@ -333,7 +333,7 @@ to and from a iteration list to manipulate the iteration of the dragged issues. ### Group issues in swimlanes **(PREMIUM)** -> - Grouping by epic [introduced](https://gitlab.com/groups/gitlab-org/-/epics/3352) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.6. +> - Grouping by epic [introduced](https://gitlab.com/groups/gitlab-org/-/epics/3352) in GitLab 13.6. > - Editing issue titles in the issue sidebar [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232745) in GitLab 13.8. > - Editing iteration in the issue sidebar [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/290232) in GitLab 13.9. @@ -356,7 +356,7 @@ appears on the right. There you can see and edit the issue's: - Title - Assignees -- Epic **(PREMIUM)** +- [Epic](../group/epics/index.md) - Milestone - Time tracking value (view only) - Due date @@ -506,14 +506,14 @@ You can filter by the following: - Assignee - Author -- Epic -- Iteration +- [Epic](../group/epics/index.md) +- [Iteration](../group/iterations/index.md) - Label - Milestone - My Reaction - Release - Type (issue/incident) -- Weight +- [Weight](issues/issue_weight.md) #### Filtering issues in a group board diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 7ccc39eeb8b..45ed5960626 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -13,9 +13,9 @@ so you can track and find the work items you're interested in. Labels are a key part of [issue boards](issue_board.md). With labels you can: -- Categorize epics, issues, and merge requests using colors and descriptive titles like +- Categorize [epics](../group/epics/index.md), issues, and merge requests using colors and descriptive titles like `bug`, `feature request`, or `docs`. -- Dynamically filter and manage epics, issues, and merge requests. +- Dynamically filter and manage [epics](../group/epics/index.md), issues, and merge requests. - [Search lists of issues, merge requests, and epics](../search/index.md#search-issues-and-merge-requests), as well as [issue boards](../search/index.md#issue-boards). @@ -26,7 +26,7 @@ There are two types of labels in GitLab: - **Project labels** can be assigned to issues and merge requests in that project only. - **Group labels** can be assigned to issues and merge requests in any project in the selected group or its subgroups. - - They can also be assigned to epics in the selected group or its subgroups.**(ULTIMATE)** + - They can also be assigned to [epics](../group/epics/index.md) in the selected group or its subgroups. ## Assign and unassign labels diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 9e08ce9dd88..ae42d90af06 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -7,11 +7,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab quick actions **(FREE)** -> - Introduced in [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672): +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26672) in GitLab 12.1: > once an action is executed, an alert appears when a quick action is successfully applied. -> - Introduced in [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/16877): you can use +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16877) in GitLab 13.2: you can use > quick actions when updating the description of issues, epics, and merge requests. -> - Introduced in [GitLab 13.8](https://gitlab.com/gitlab-org/gitlab/-/issues/292393): when you enter +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292393) in GitLab 13.8: when you enter > `/` into a description or comment field, all available quick actions are displayed in a scrollable list. > - The rebase quick action was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49800) in GitLab 13.8. @@ -49,15 +49,15 @@ threads. Some quick actions might not be available to all subscription tiers. | Command | Issue | Merge request | Epic | Action | |:-------------------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `/add_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add one or more [CRM contacts](../crm/index.md) ([introduced in GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413)). | +| `/add_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add one or more [CRM contacts](../crm/index.md) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413) in GitLab 14.6). | | `/approve` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Approve the merge request. | | `/assign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users. | | `/assign me` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself. | | `/assign_reviewer @user1 @user2` or `/reviewer @user1 @user2` or `/request_review @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users as reviewers. | | `/assign_reviewer me` or `/reviewer me` or `/request_review me` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself as a reviewer. | | `/award :emoji:` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Toggle emoji award. | -| `/child_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Add child epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/7330)). | -| `/clear_health_status` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear [health status](issues/managing_issues.md#health-status) ([introduced in GitLab 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/213814)). | +| `/child_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Add child epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7330) in GitLab 12.0). | +| `/clear_health_status` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear [health status](issues/managing_issues.md#health-status) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). | | `/clear_weight` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear weight. | | `/clone <path/to/project> [--with_notes]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clone the issue to given project, or the current one if no arguments are given ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9421) in GitLab 13.7). Copies as much data as possible as long as the target project contains equivalent labels, milestones, and so on. Does not copy comments or system notes unless `--with_notes` is provided as an argument. | | `/close` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Close. | @@ -68,48 +68,48 @@ threads. Some quick actions might not be available to all subscription tiers. | `/done` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mark to do as done. | | `/draft` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Toggle the draft status. | | `/due <date>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | -| `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue and mark as a duplicate of another issue. Also, mark both as related. | +| `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue and mark as a duplicate of another issue. Also, mark both as related. | | `/epic <epic>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. | | `/estimate <time>` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Set time estimate. For example, `/estimate 1mo 2w 3d 4h 5m`. Learn more about [time tracking](time_tracking.md). | -| `/health_status <value>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set [health status](issues/managing_issues.md#health-status). Valid options for `<value>` are `on_track`, `needs_attention`, and `at_risk` ([introduced in GitLab 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/213814)). | +| `/health_status <value>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set [health status](issues/managing_issues.md#health-status). Valid options for `<value>` are `on_track`, `needs_attention`, and `at_risk` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). | | `/invite_email email1 email2` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add up to six email participants. This action is behind feature flag `issue_email_participants` and is not yet supported in issue templates. | -| `/iteration *iteration:"iteration name"` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)). | +| `/iteration *iteration:"iteration name"` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set iteration. For example, to set the `Late in July` iteration: `/iteration *iteration:"Late in July"` ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196795) in GitLab 13.1). | | `/label ~label1 ~label2` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Add one or more labels. Label names can also start without a tilde (`~`), but mixed syntax is not supported. | | `/lock` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Lock the discussions. | | `/merge` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Merge changes. Depending on the project setting, this may be [when the pipeline succeeds](merge_requests/merge_when_pipeline_succeeds.md), or adding to a [Merge Train](../../ci/pipelines/merge_trains.md). | | `/milestone %milestone` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Set milestone. | -| `/move <path/to/project>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Move this issue to another project. Be careful when moving an issue to a project with different access rules. Before moving the issue, make sure it does not contain sensitive data. | -| `/parent_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). | +| `/move <path/to/project>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Move this issue to another project. Be careful when moving an issue to a project with different access rules. Before moving the issue, make sure it does not contain sensitive data. | +| `/parent_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Set parent epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10556) in GitLab 12.1). | | `/promote` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Promote issue to epic. | -| `/promote_to_incident` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Promote issue to incident ([introduced in GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/296787)). | -| `/page <policy name>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Start escalations for the incident ([introduced in GitLab 14.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79977)). -| `/publish` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Publish issue to an associated [Status Page](../../operations/incident_management/status_page.md) ([Introduced in GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906)) | +| `/promote_to_incident` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Promote issue to incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296787) in GitLab 14.5). | +| `/page <policy name>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Start escalations for the incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79977) in GitLab 14.9). | +| `/publish` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Publish issue to an associated [Status Page](../../operations/incident_management/status_page.md) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906) in GitLab 13.0) | | `/reassign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Replace current assignees with those specified. | | `/reassign_reviewer @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Replace current reviewers with those specified. | | `/rebase` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Rebase source branch. This schedules a background task that attempts to rebase the changes in the source branch on the latest commit of the target branch. If `/rebase` is used, `/merge` is ignored to avoid a race condition where the source branch is merged or deleted before it is rebased. If there are merge conflicts, GitLab displays a message that a rebase cannot be scheduled. Rebase failures are displayed with the merge request status. | | `/relabel ~label1 ~label2` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Replace current labels with those specified. | | `/relate #issue1 #issue2` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Mark issues as related. | -| `/remove_child_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Remove child epic from `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced in GitLab 12.0](https://gitlab.com/gitlab-org/gitlab/-/issues/7330)). | -| `/remove_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove one or more [CRM contacts](../crm/index.md) ([introduced in GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413)). | +| `/remove_child_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Remove child epic from `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7330) in GitLab 12.0). | +| `/remove_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove one or more [CRM contacts](../crm/index.md) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413) in GitLab 14.6). | | `/remove_due_date` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove due date. | | `/remove_epic` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove from epic. | | `/remove_estimate` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Remove time estimate. | -| `/remove_iteration` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove iteration ([introduced in GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/issues/196795)). | +| `/remove_iteration` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove iteration ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196795) in GitLab 13.1). | | `/remove_milestone` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Remove milestone. | -| `/remove_parent_epic` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Remove parent epic from epic ([introduced in GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/issues/10556)). | +| `/remove_parent_epic` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Remove parent epic from epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10556) in GitLab 12.1). | | `/remove_time_spent` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Remove time spent. | -| `/remove_zoom` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove Zoom meeting from this issue ([introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)). | +| `/remove_zoom` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Remove Zoom meeting from this issue ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609) in GitLab 12.4). | | `/reopen` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Reopen. | -| `/severity <severity>` | **{check-circle}** Yes | **{check-circle}** No | **{check-circle}** No | Set the severity. Options for `<severity>` are `S1` ... `S4`, `critical`, `high`, `medium`, `low`, `unknown`. [Introduced in GitLab 14.2](https://gitlab.com/gitlab-org/gitlab/-/issues/334045). | +| `/severity <severity>` | **{check-circle}** Yes | **{check-circle}** No | **{check-circle}** No | Set the severity. Options for `<severity>` are `S1` ... `S4`, `critical`, `high`, `medium`, `low`, `unknown`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334045) in GitLab 14.2. | | `/shrug <comment>` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Append the comment with `¯\＿(ツ)＿/¯`. | | `/spend <time> [<date>]` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Add or subtract spent time. Optionally, specify the date that time was spent on. For example, `/spend 1mo 2w 3d 4h 5m 2018-08-26` or `/spend -1h 30m`. Learn more about [time tracking](time_tracking.md). | -| `/submit_review` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Submit a pending review ([introduced in GitLab 12.7](https://gitlab.com/gitlab-org/gitlab/-/issues/8041)). | +| `/submit_review` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Submit a pending review ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8041) in GitLab 12.7). | | `/subscribe` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Subscribe to notifications. | | `/tableflip <comment>` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Append the comment with `(╯°□°)╯︵ ┻━┻`. | | `/target_branch <local branch name>` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Set target branch. | | `/title <new title>` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Change title. | | `/todo` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Add a to-do item. | -| `/unapprove` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Unapprove the merge request. ([introduced in GitLab 14.3](https://gitlab.com/gitlab-org/gitlab/-/issues/8103) | +| `/unapprove` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Unapprove the merge request. ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8103) in GitLab 14.3 | | `/unassign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Remove specific assignees. | | `/unassign` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Remove all assignees. | | `/unassign_reviewer @user1 @user2` or `/remove_reviewer @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Remove specific reviewers. | @@ -119,7 +119,7 @@ threads. Some quick actions might not be available to all subscription tiers. | `/unlock` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Unlock the discussions. | | `/unsubscribe` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Unsubscribe from notifications. | | `/weight <value>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set weight. Valid options for `<value>` include `0`, `1`, `2`, and so on. | -| `/zoom <Zoom URL>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add Zoom meeting to this issue ([introduced in GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609)). | +| `/zoom <Zoom URL>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add Zoom meeting to this issue ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16609) in GitLab 12.4). | ## Commit messages diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 66cf39b2634..b7fe5a2be29 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -93,8 +93,8 @@ displayed in the information note. ### Using customized email templates -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2460) in GitLab Premium 12.7. -> - Moved to GitLab Free in 13.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2460) in GitLab 12.7. +> - Moved from GitLab Premium to GitLab Free in 13.2. An email is sent to the author when: @@ -171,7 +171,7 @@ To edit the custom email display name: ### Using a custom email address **(FREE SELF)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2201) in GitLab Premium 13.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2201) in GitLab 13.0. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/284656) in GitLab 13.8. It is possible to customize the email address used by Service Desk. To do this, you must configure @@ -241,7 +241,7 @@ The configuration options are the same as for configuring ##### Microsoft Graph -> Introduced in [GitLab 13.11](https://gitlab.com/gitlab-org/gitlab/-/issues/214900) +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214900) in GitLab 13.11. Service Desk can be configured to read Microsoft Exchange Online mailboxes with the Microsoft Graph API instead of IMAP. Follow the [documentation in the incoming email section for setting up an OAuth2 application for Microsoft Graph](../../administration/incoming_email.md#microsoft-graph). diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index d5bf0cffb1e..a1918ee6ad5 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -67,7 +67,10 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) - Gitlab::ApplicationContext.push(feature_category: route.feature_category) + Gitlab::ApplicationContext.push( + feature_category: route.feature_category, + caller_id: route.caller_id + ) new_headers = { 'ETag' => etag, diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 742b72ecde9..684afc6762a 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,22 +3,33 @@ module Gitlab module EtagCaching module Router - Route = Struct.new(:regexp, :name, :feature_category, :router) do + Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id) do delegate :match, to: :regexp delegate :cache_key, to: :router end module Helpers def build_route(attrs) - EtagCaching::Router::Route.new(*attrs, self) + EtagCaching::Router::Route.new(self, *attrs) + end + + def build_rails_route(attrs) + regexp, name, controller, action_name = *attrs + EtagCaching::Router::Route.new( + self, + regexp, + name, + controller.feature_category_for_action(action_name).to_s, + controller.endpoint_id_for_action(action_name).to_s + ) end end - # Performing RESTful routing match before GraphQL would be more expensive + # Performing Rails routing match before GraphQL would be more expensive # for the GraphQL requests because we need to traverse all of the RESTful # route definitions before falling back to GraphQL. def self.match(request) - Router::Graphql.match(request) || Router::Restful.match(request) + Router::Graphql.match(request) || Router::Rails.match(request) end end end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/rails.rb index 176676bd6ba..d80c003fe53 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -3,7 +3,7 @@ module Gitlab module EtagCaching module Router - class Restful + class Rails extend EtagCaching::Router::Helpers # We enable an ETag for every request matching the regex. @@ -23,74 +23,88 @@ module Gitlab [ %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), 'issue_notes', - 'team_planning' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), 'merge_request_notes', - 'code_review' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), 'issue_title', - 'team_planning' + ::Projects::IssuesController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), 'commit_pipelines', - 'continuous_integration' + ::Projects::CommitController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), 'new_merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequests::CreationsController, + :new ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequestsController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), 'project_pipelines', - 'continuous_integration' + ::Projects::PipelinesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), 'project_pipeline', - 'continuous_integration' + ::Projects::PipelinesController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), 'project_build', - 'continuous_integration' + ::Projects::BuildsController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), 'cluster_environments', - 'continuous_delivery' + ::Groups::ClustersController, + :environments ], [ %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), 'environments', - 'continuous_delivery' + ::Projects::EnvironmentsController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), 'realtime_changes_import_github', - 'importers' + ::Import::GithubController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea', - 'importers' + ::Import::GiteaController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), 'merge_request_widget', - 'code_review' + ::Projects::MergeRequests::ContentController, + :cached_widget ] - ].map(&method(:build_route)).freeze + ].map(&method(:build_rails_route)).freeze # Overridden in EE to add more routes def self.all_routes @@ -109,4 +123,4 @@ module Gitlab end end -Gitlab::EtagCaching::Router::Restful.prepend_mod_with('Gitlab::EtagCaching::Router::Restful') +Gitlab::EtagCaching::Router::Rails.prepend_mod_with('Gitlab::EtagCaching::Router::Rails') diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 02b582190b6..a9f8483d8c3 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class DiffNoteImporter + DiffNoteCreationError = Class.new(ActiveRecord::RecordInvalid) + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -31,7 +33,7 @@ module Gitlab else import_with_legacy_diff_note end - rescue ::DiffNote::NoteDiffFileCreationError => e + rescue ::DiffNote::NoteDiffFileCreationError, DiffNoteCreationError => e Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note @@ -84,7 +86,7 @@ module Gitlab def import_with_diff_note log_diff_note_creation('DiffNote') - ::Import::Github::Notes::CreateService.new(project, author, { + record = ::Import::Github::Notes::CreateService.new(project, author, { noteable_type: note.noteable_type, system: false, type: 'DiffNote', @@ -97,6 +99,8 @@ module Gitlab updated_at: note.updated_at, position: note.diff_position }).execute + + raise DiffNoteCreationError, record unless record.persisted? end def note_body diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2bd59415771..d59be09f85e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -52,7 +52,6 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 96755db8439..fdf4bc58525 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -324,7 +324,6 @@ category: snippets redis_slot: snippets aggregation: weekly - feature_flag: usage_data_i_snippets_show # Terraform - name: p_terraform_state_api_unique_users category: terraform diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 7000ea65eaf..64cc883dc68 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -43568,7 +43568,7 @@ msgstr "" msgid "is not a valid X509 certificate." msgstr "" -msgid "is not allowed for sign-up." +msgid "is not allowed for sign-up. Please use your regular email address." msgstr "" msgid "is not allowed for this group." @@ -43580,7 +43580,7 @@ msgstr "" msgid "is not allowed since the group is not top-level group." msgstr "" -msgid "is not allowed." +msgid "is not allowed. Please use your regular email address." msgstr "" msgid "is not allowed. We do not currently support project-level iterations" diff --git a/rubocop/cop/gitlab/avoid_uploaded_file_from_params.rb b/rubocop/cop/gitlab/avoid_uploaded_file_from_params.rb index 599371aa5a1..0795f96732d 100644 --- a/rubocop/cop/gitlab/avoid_uploaded_file_from_params.rb +++ b/rubocop/cop/gitlab/avoid_uploaded_file_from_params.rb @@ -4,7 +4,7 @@ module RuboCop module Cop module Gitlab # This cop checks for `UploadedFile.from_params` usage. - # See https://docs.gitlab.com/ee/development/uploads.html#how-to-add-a-new-upload-route + # See https://docs.gitlab.com/ee/development/uploads/working_with_uploads.html # # @example # @@ -34,7 +34,7 @@ module RuboCop # end # end class AvoidUploadedFileFromParams < RuboCop::Cop::Cop - MSG = 'Use the `UploadedFile` set by `multipart.rb` instead of calling `UploadedFile.from_params` directly. See https://docs.gitlab.com/ee/development/uploads.html#how-to-add-a-new-upload-route' + MSG = 'Use the `UploadedFile` set by `multipart.rb` instead of calling `UploadedFile.from_params` directly. See https://docs.gitlab.com/ee/development/uploads/working_with_uploads.html' def_node_matcher :calling_uploaded_file_from_params?, <<~PATTERN (send (const nil? :UploadedFile) :from_params ...) diff --git a/spec/finders/releases_finder_spec.rb b/spec/finders/releases_finder_spec.rb index 5ddb5c33fad..b0fa1177245 100644 --- a/spec/finders/releases_finder_spec.rb +++ b/spec/finders/releases_finder_spec.rb @@ -33,18 +33,6 @@ RSpec.describe ReleasesFinder do end end - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27716 - shared_examples_for 'when tag is nil' do - before do - v1_0_0.update_column(:tag, nil) - end - - it 'ignores rows with a nil tag' do - expect(subject.size).to eq(1) - expect(subject).to eq([v1_1_0]) - end - end - shared_examples_for 'when a tag parameter is passed' do let(:params) { { tag: 'v1.0.0' } } @@ -116,7 +104,6 @@ RSpec.describe ReleasesFinder do end it_behaves_like 'preload' - it_behaves_like 'when tag is nil' it_behaves_like 'when a tag parameter is passed' end end diff --git a/spec/frontend/security_configuration/components/feature_card_spec.js b/spec/frontend/security_configuration/components/feature_card_spec.js index 2b74be19480..f0d902bf9fe 100644 --- a/spec/frontend/security_configuration/components/feature_card_spec.js +++ b/spec/frontend/security_configuration/components/feature_card_spec.js @@ -50,7 +50,7 @@ describe('FeatureCard component', () => { expect(enableLinks.exists()).toBe(expectEnableAction); if (expectEnableAction) { expect(enableLinks).toHaveLength(1); - expect(enableLinks.at(0).props('category')).toBe('primary'); + expect(enableLinks.at(0).props('category')).toBe('secondary'); } const configureLinks = findConfigureLinks(); diff --git a/spec/lib/gitlab/etag_caching/middleware_spec.rb b/spec/lib/gitlab/etag_caching/middleware_spec.rb index 982c0d911bc..8228f95dd5e 100644 --- a/spec/lib/gitlab/etag_caching/middleware_spec.rb +++ b/spec/lib/gitlab/etag_caching/middleware_spec.rb @@ -174,7 +174,8 @@ RSpec.describe Gitlab::EtagCaching::Middleware, :clean_gitlab_redis_shared_state it "pushes route's feature category to the context" do expect(Gitlab::ApplicationContext).to receive(:push).with( - feature_category: 'team_planning' + feature_category: 'team_planning', + caller_id: 'Projects::NotesController#index' ) _, _, _ = middleware.call(build_request(path, if_none_match)) diff --git a/spec/lib/gitlab/etag_caching/router/restful_spec.rb b/spec/lib/gitlab/etag_caching/router/rails_spec.rb index a0fc480369c..da6c11e3cb1 100644 --- a/spec/lib/gitlab/etag_caching/router/restful_spec.rb +++ b/spec/lib/gitlab/etag_caching/router/rails_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::EtagCaching::Router::Restful do +RSpec.describe Gitlab::EtagCaching::Router::Rails do it 'matches issue notes endpoint' do result = match_route('/my-group/and-subgroup/here-comes-the-project/noteable/issue/1/notes') @@ -114,6 +114,12 @@ RSpec.describe Gitlab::EtagCaching::Router::Restful do end end + it 'has a caller_id for every route', :aggregate_failures do + described_class::ROUTES.each do |route| + expect(route.caller_id).to include('#'), "#{route.name} has caller_id #{route.caller_id}, which is not valid" + end + end + def match_route(path) described_class.match(double(path_info: path)) end diff --git a/spec/lib/gitlab/etag_caching/router_spec.rb b/spec/lib/gitlab/etag_caching/router_spec.rb index ce728c41f48..8d2183bc03d 100644 --- a/spec/lib/gitlab/etag_caching/router_spec.rb +++ b/spec/lib/gitlab/etag_caching/router_spec.rb @@ -10,7 +10,7 @@ RSpec.describe Gitlab::EtagCaching::Router do expect(result).to be_present expect(result.name).to eq 'project_pipelines' - expect(result.router).to eq Gitlab::EtagCaching::Router::Restful + expect(result.router).to eq Gitlab::EtagCaching::Router::Rails end end diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index c8e744ab262..321ad7d3238 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -12,6 +12,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail let(:updated_at) { Time.new(2017, 1, 1, 12, 15).utc } let(:note_body) { 'Hello' } let(:file_path) { 'files/ruby/popen.rb' } + let(:end_line) { 15 } let(:diff_hunk) do '@@ -14 +14 @@ @@ -31,7 +32,7 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail created_at: created_at, updated_at: updated_at, start_line: nil, - end_line: 15, + end_line: end_line, github_id: 1, diff_hunk: diff_hunk, side: 'RIGHT' @@ -173,7 +174,24 @@ RSpec.describe Gitlab::GithubImport::Importer::DiffNoteImporter, :aggregate_fail NOTE end - context 'when the note diff file creation fails' do + context 'when the note diff file creation fails with DiffNoteCreationError due to outdated suggestion' do + let(:end_line) { nil } + + it 'falls back to the LegacyDiffNote' do + expect(Gitlab::GithubImport::Logger) + .to receive(:warn) + .with( + message: "Validation failed: Line code can't be blank, Line code must be a valid line code, Position is incomplete", + 'error.class': 'Gitlab::GithubImport::Importer::DiffNoteImporter::DiffNoteCreationError' + ) + + expect { subject.execute } + .to change(LegacyDiffNote, :count) + .and not_change(DiffNote, :count) + end + end + + context 'when the note diff file creation fails with NoteDiffFileCreationError' do it 'falls back to the LegacyDiffNote' do exception = ::DiffNote::NoteDiffFileCreationError.new('Failed to create diff note file') diff --git a/spec/migrations/20220222192524_create_not_null_constraint_releases_tag_spec.rb b/spec/migrations/20220222192524_create_not_null_constraint_releases_tag_spec.rb new file mode 100644 index 00000000000..bd7d992240a --- /dev/null +++ b/spec/migrations/20220222192524_create_not_null_constraint_releases_tag_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +require 'spec_helper' +require_migration! + +RSpec.describe CreateNotNullConstraintReleasesTag do + let_it_be(:releases) { table(:releases) } + let_it_be(:migration) { described_class.new } + + before do + allow(migration).to receive(:transaction_open?).and_return(false) + allow(migration).to receive(:with_lock_retries).and_yield + end + + it 'adds a check constraint to tags' do + constraint = releases.connection.check_constraints(:releases).find { |constraint| constraint.expression == "tag IS NOT NULL" } + expect(constraint).to be_nil + + migration.up + + constraint = releases.connection.check_constraints(:releases).find { |constraint| constraint.expression == "tag IS NOT NULL" } + expect(constraint).to be_a(ActiveRecord::ConnectionAdapters::CheckConstraintDefinition) + end +end diff --git a/spec/migrations/20220222192525_remove_null_releases_spec.rb b/spec/migrations/20220222192525_remove_null_releases_spec.rb new file mode 100644 index 00000000000..6043f2c8cc8 --- /dev/null +++ b/spec/migrations/20220222192525_remove_null_releases_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +require 'spec_helper' + +require_migration! + +RSpec.describe RemoveNullReleases do + let(:releases) { table(:releases) } + + before do + # we need to migrate to before previous migration so an invalid record can be created + migrate! + migration_context.down(previous_migration(3).version) + + releases.create!(tag: 'good', name: 'good release', released_at: Time.now) + releases.create!(tag: nil, name: 'bad release', released_at: Time.now) + end + + it 'deletes template records and associated data' do + expect { migrate! } + .to change { releases.count }.from(2).to(1) + end +end diff --git a/spec/models/preloaders/environments/deployment_preloader_spec.rb b/spec/models/preloaders/environments/deployment_preloader_spec.rb index c1812d45628..3f2f28a069e 100644 --- a/spec/models/preloaders/environments/deployment_preloader_spec.rb +++ b/spec/models/preloaders/environments/deployment_preloader_spec.rb @@ -62,4 +62,22 @@ RSpec.describe Preloaders::Environments::DeploymentPreloader do expect(default_preload_query).to be(false) end + + it 'sets environment on the associated deployment', :aggregate_failures do + preload_association(:last_deployment) + + expect do + project.environments.each { |environment| environment.last_deployment.environment } + end.not_to exceed_query_limit(0) + + project.environments.each do |environment| + expect(environment.last_deployment.environment).to eq(environment) + end + end + + it 'does not attempt to set environment on a nil deployment' do + create(:environment, project: project, state: :available) + + expect { preload_association(:last_deployment) }.not_to raise_error + end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 6e7cccfccf6..a5bf32972a2 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -498,7 +498,7 @@ RSpec.describe User do end describe 'email' do - let(:expected_error) { _('is not allowed for sign-up. Check with your administrator.') } + let(:expected_error) { _('is not allowed for sign-up. Please use your regular email address. Check with your administrator.') } context 'when no signup domains allowed' do before do @@ -550,7 +550,7 @@ RSpec.describe User do user = create(:user, email: "info@test.example.com") expect { user.update!(email: "test@notexample.com") } - .to raise_error(StandardError, 'Validation failed: Email is not allowed. Check with your administrator.') + .to raise_error(StandardError, 'Validation failed: Email is not allowed. Please use your regular email address. Check with your administrator.') end end @@ -623,7 +623,7 @@ RSpec.describe User do user = create(:user, email: 'info@test.com') expect { user.update!(email: 'info@example.com') } - .to raise_error(StandardError, 'Validation failed: Email is not allowed. Check with your administrator.') + .to raise_error(StandardError, 'Validation failed: Email is not allowed. Please use your regular email address. Check with your administrator.') end end @@ -700,7 +700,7 @@ RSpec.describe User do user = create(:user, email: 'info@test.com') expect { user.update!(email: 'info@gitlab.com') } - .to raise_error(StandardError, 'Validation failed: Email is not allowed. Check with your administrator.') + .to raise_error(StandardError, 'Validation failed: Email is not allowed. Please use your regular email address. Check with your administrator.') end it 'does accept a valid email address' do diff --git a/spec/support/helpers/migrations_helpers.rb b/spec/support/helpers/migrations_helpers.rb index 0c5bf09f6b7..e7dd2d54aee 100644 --- a/spec/support/helpers/migrations_helpers.rb +++ b/spec/support/helpers/migrations_helpers.rb @@ -104,9 +104,9 @@ module MigrationsHelpers stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') end - def previous_migration - migrations.each_cons(2) do |previous, migration| - break previous if migration.name == described_class.name + def previous_migration(steps_back = 2) + migrations.each_cons(steps_back) do |cons| + break cons.first if cons.last.name == described_class.name end end diff --git a/spec/support/shared_examples/serializers/environment_serializer_shared_examples.rb b/spec/support/shared_examples/serializers/environment_serializer_shared_examples.rb index ad4057a972a..87a33060435 100644 --- a/spec/support/shared_examples/serializers/environment_serializer_shared_examples.rb +++ b/spec/support/shared_examples/serializers/environment_serializer_shared_examples.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true RSpec.shared_examples 'avoid N+1 on environments serialization' do |ee: false| # Investigating in https://gitlab.com/gitlab-org/gitlab/-/issues/353209 - let(:query_threshold) { 9 + (ee ? 4 : 0) } + let(:query_threshold) { 1 + (ee ? 4 : 0) } it 'avoids N+1 database queries with grouping', :request_store do create_environment_with_associations(project) |