diff options
33 files changed, 708 insertions, 217 deletions
diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 04a36953d66..b6473322b1f 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -277,7 +277,8 @@ "type": "string", "format": "uri-reference", "pattern": "\\.ya?ml$" - } + }, + "rules": { "$ref": "#/definitions/rules" } }, "required": ["local"] }, @@ -491,7 +492,7 @@ }, "rules": { "type": "array", - "description": "rules allows for an array of individual rule objects to be evaluated in order, until one matches and dynamically provides attributes to the job.", + "description": "Rules allows for an array of individual rule objects to be evaluated in order, until one matches and dynamically provides attributes to the job.", "items": { "type": "object", "additionalProperties": false, @@ -625,17 +626,26 @@ ] }, "cache": { - "oneOf": [ - { - "$ref": "#/definitions/cache_entry" - }, - { - "type": "array", - "items": { - "$ref": "#/definitions/cache_entry" - } + "properties": { + "when": { + "description": "Defines when to save the cache, based on the status of the job.", + "default": "on_success", + "oneOf": [ + { + "enum": ["on_success"], + "description": "Save the cache only when the job succeeds." + }, + { + "enum": ["on_failure"], + "description": "Save the cache only when the job fails. " + }, + { + "enum": ["always"], + "description": "Always save the cache. " + } + ] } - ] + } }, "cache_entry": { "type": "object", @@ -1320,6 +1330,29 @@ } }, "required": ["artifact", "job"] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "project": { + "description": "Path to another private project under the same GitLab instance, like `group/project` or `group/sub-group/project`.", + "type": "string", + "pattern": "\\S/\\S" + }, + "ref": { + "description": "Branch/Tag/Commit hash for the target project.", + "minLength": 1, + "type": "string" + }, + "file": { + "description": "Relative path from repository root (`/`) to the pipeline configuration YAML file.", + "type": "string", + "format": "uri-reference", + "pattern": "\\.ya?ml$" + } + }, + "required": ["project", "file"] } ] } diff --git a/app/assets/javascripts/issues_list/components/issues_list_app.vue b/app/assets/javascripts/issues_list/components/issues_list_app.vue index 8e37339fca6..de57cb93899 100644 --- a/app/assets/javascripts/issues_list/components/issues_list_app.vue +++ b/app/assets/javascripts/issues_list/components/issues_list_app.vue @@ -82,6 +82,7 @@ import searchLabelsQuery from '../queries/search_labels.query.graphql'; import searchMilestonesQuery from '../queries/search_milestones.query.graphql'; import searchUsersQuery from '../queries/search_users.query.graphql'; import IssueCardTimeInfo from './issue_card_time_info.vue'; +import NewIssueDropdown from './new_issue_dropdown.vue'; export default { i18n, @@ -96,6 +97,7 @@ export default { IssuableByEmail, IssuableList, IssueCardTimeInfo, + NewIssueDropdown, BlockingIssuesCount: () => import('ee_component/issues/components/blocking_issues_count.vue'), }, directives: { @@ -126,6 +128,9 @@ export default { hasAnyIssues: { default: false, }, + hasAnyProjects: { + default: false, + }, hasBlockedIssuesFeature: { default: false, }, @@ -253,6 +258,9 @@ export default { showCsvButtons() { return this.isProject && this.isSignedIn; }, + showNewIssueDropdown() { + return !this.isProject && this.hasAnyProjects; + }, apiFilterParams() { return convertToApiParams(this.filterTokens); }, @@ -662,6 +670,7 @@ export default { <gl-button v-if="showNewIssueLink" :href="newIssuePath" variant="confirm"> {{ $options.i18n.newIssueLabel }} </gl-button> + <new-issue-dropdown v-if="showNewIssueDropdown" /> </template> <template #timeframe="{ issuable = {} }"> @@ -765,6 +774,7 @@ export default { :export-csv-path="exportCsvPathWithQuery" :issuable-count="currentTabCount" /> + <new-issue-dropdown v-if="showNewIssueDropdown" /> </template> </gl-empty-state> <hr /> diff --git a/app/assets/javascripts/issues_list/components/new_issue_dropdown.vue b/app/assets/javascripts/issues_list/components/new_issue_dropdown.vue new file mode 100644 index 00000000000..037fd9be542 --- /dev/null +++ b/app/assets/javascripts/issues_list/components/new_issue_dropdown.vue @@ -0,0 +1,124 @@ +<script> +import { + GlDropdown, + GlDropdownItem, + GlDropdownText, + GlLoadingIcon, + GlSearchBoxByType, +} from '@gitlab/ui'; +import createFlash from '~/flash'; +import searchProjectsQuery from '~/issues_list/queries/search_projects.query.graphql'; +import { DASH_SCOPE, joinPaths } from '~/lib/utils/url_utility'; +import { __, sprintf } from '~/locale'; +import { DEBOUNCE_DELAY } from '~/vue_shared/components/filtered_search_bar/constants'; + +export default { + i18n: { + defaultDropdownText: __('Select project to create issue'), + noMatchesFound: __('No matches found'), + toggleButtonLabel: __('Toggle project select'), + }, + components: { + GlDropdown, + GlDropdownItem, + GlDropdownText, + GlLoadingIcon, + GlSearchBoxByType, + }, + inject: ['fullPath'], + data() { + return { + projects: [], + search: '', + selectedProject: {}, + shouldSkipQuery: true, + }; + }, + apollo: { + projects: { + query: searchProjectsQuery, + variables() { + return { + fullPath: this.fullPath, + search: this.search, + }; + }, + update: ({ group }) => group.projects.nodes ?? [], + error(error) { + createFlash({ + message: __('An error occurred while loading projects.'), + captureError: true, + error, + }); + }, + skip() { + return this.shouldSkipQuery; + }, + debounce: DEBOUNCE_DELAY, + }, + }, + computed: { + dropdownHref() { + return this.hasSelectedProject + ? joinPaths(this.selectedProject.webUrl, DASH_SCOPE, 'issues/new') + : undefined; + }, + dropdownText() { + return this.hasSelectedProject + ? sprintf(__('New issue in %{project}'), { project: this.selectedProject.name }) + : this.$options.i18n.defaultDropdownText; + }, + hasSelectedProject() { + return this.selectedProject.id; + }, + showNoSearchResultsText() { + return !this.projects.length && this.search; + }, + }, + methods: { + handleDropdownClick() { + if (!this.dropdownHref) { + this.$refs.dropdown.show(); + } + }, + handleDropdownShown() { + if (this.shouldSkipQuery) { + this.shouldSkipQuery = false; + } + this.$refs.search.focusInput(); + }, + selectProject(project) { + this.selectedProject = project; + }, + }, +}; +</script> + +<template> + <gl-dropdown + ref="dropdown" + right + split + :split-href="dropdownHref" + :text="dropdownText" + :toggle-text="$options.i18n.toggleButtonLabel" + variant="confirm" + @click="handleDropdownClick" + @shown="handleDropdownShown" + > + <gl-search-box-by-type ref="search" v-model.trim="search" /> + <gl-loading-icon v-if="$apollo.queries.projects.loading" /> + <template v-else> + <gl-dropdown-item + v-for="project of projects" + :key="project.id" + @click="selectProject(project)" + > + {{ project.nameWithNamespace }} + </gl-dropdown-item> + <gl-dropdown-text v-if="showNoSearchResultsText"> + {{ $options.i18n.noMatchesFound }} + </gl-dropdown-text> + </template> + </gl-dropdown> +</template> diff --git a/app/assets/javascripts/issues_list/index.js b/app/assets/javascripts/issues_list/index.js index e89e3e8e681..62180c7ffa4 100644 --- a/app/assets/javascripts/issues_list/index.js +++ b/app/assets/javascripts/issues_list/index.js @@ -121,6 +121,7 @@ export function mountIssuesListApp() { fullPath, groupEpicsPath, hasAnyIssues, + hasAnyProjects, hasBlockedIssuesFeature, hasIssuableHealthStatusFeature, hasIssueWeightsFeature, @@ -153,6 +154,7 @@ export function mountIssuesListApp() { fullPath, groupEpicsPath, hasAnyIssues: parseBoolean(hasAnyIssues), + hasAnyProjects: parseBoolean(hasAnyProjects), hasBlockedIssuesFeature: parseBoolean(hasBlockedIssuesFeature), hasIssuableHealthStatusFeature: parseBoolean(hasIssuableHealthStatusFeature), hasIssueWeightsFeature: parseBoolean(hasIssueWeightsFeature), diff --git a/app/assets/javascripts/issues_list/queries/search_projects.query.graphql b/app/assets/javascripts/issues_list/queries/search_projects.query.graphql new file mode 100644 index 00000000000..df1f330139a --- /dev/null +++ b/app/assets/javascripts/issues_list/queries/search_projects.query.graphql @@ -0,0 +1,12 @@ +query searchProjects($fullPath: ID!, $search: String) { + group(fullPath: $fullPath) { + projects(search: $search, includeSubgroups: true) { + nodes { + id + name + nameWithNamespace + webUrl + } + } + } +} diff --git a/app/assets/javascripts/lib/utils/url_utility.js b/app/assets/javascripts/lib/utils/url_utility.js index bca0e45d98d..6580a028e4a 100644 --- a/app/assets/javascripts/lib/utils/url_utility.js +++ b/app/assets/javascripts/lib/utils/url_utility.js @@ -1,3 +1,5 @@ +export const DASH_SCOPE = '-'; + const PATH_SEPARATOR = '/'; const PATH_SEPARATOR_LEADING_REGEX = new RegExp(`^${PATH_SEPARATOR}+`); const PATH_SEPARATOR_ENDING_REGEX = new RegExp(`${PATH_SEPARATOR}+$`); diff --git a/app/assets/stylesheets/framework/blocks.scss b/app/assets/stylesheets/framework/blocks.scss index a0682eabf01..4b84dae5355 100644 --- a/app/assets/stylesheets/framework/blocks.scss +++ b/app/assets/stylesheets/framework/blocks.scss @@ -322,7 +322,7 @@ display: inline-block; } - .btn { + .btn:not(.split-content-button):not(.dropdown-toggle-split) { margin: $gl-padding-8 $gl-padding-4; @include media-breakpoint-down(xs) { diff --git a/app/assets/stylesheets/startup/startup-dark.scss b/app/assets/stylesheets/startup/startup-dark.scss index 50baa91b326..7f5b61191b3 100644 --- a/app/assets/stylesheets/startup/startup-dark.scss +++ b/app/assets/stylesheets/startup/startup-dark.scss @@ -315,6 +315,9 @@ h1 { padding-left: 0.6em; border-radius: 10rem; } +.bg-transparent { + background-color: transparent !important; +} .rounded-circle { border-radius: 50% !important; } @@ -359,6 +362,20 @@ h1 { .m-auto { margin: auto !important; } +.gl-badge { + display: inline-flex; + align-items: center; + font-size: 0.75rem; + font-weight: 400; + line-height: 1rem; + padding-top: 0.25rem; + padding-bottom: 0.25rem; + padding-left: 0.5rem; + padding-right: 0.5rem; +} +.gl-button .gl-badge { + top: 0; +} .gl-form-input, .gl-form-input.form-control { background-color: #333; @@ -856,6 +873,12 @@ input { .navbar-nav .badge.badge-pill:not(.merge-request-badge).todos-count { background-color: var(--blue-400, #1f75cb); } +.title-container .canary-badge .badge, +.navbar-nav .canary-badge .badge { + font-size: 12px; + line-height: 16px; + padding: 0 0.5rem; +} @media (max-width: 575.98px) { .navbar-gitlab .container-fluid { font-size: 18px; @@ -1822,9 +1845,18 @@ body.gl-dark { white-space: nowrap; width: 1px; } +.gl-bg-green-500 { + background-color: #2da160; +} .gl-border-none\! { border-style: none !important; } +.gl-rounded-pill { + border-radius: 0.75rem; +} +.gl-text-white { + color: #333; +} .gl-display-none { display: none; } @@ -1843,6 +1875,10 @@ body.gl-dark { .gl-pr-2 { padding-right: 0.25rem; } +.gl-py-1 { + padding-top: 0.125rem; + padding-bottom: 0.125rem; +} .gl-ml-3 { margin-left: 0.5rem; } diff --git a/app/assets/stylesheets/startup/startup-general.scss b/app/assets/stylesheets/startup/startup-general.scss index e4ab6a05ff5..5dcfacdb1a8 100644 --- a/app/assets/stylesheets/startup/startup-general.scss +++ b/app/assets/stylesheets/startup/startup-general.scss @@ -295,6 +295,9 @@ h1 { padding-left: 0.6em; border-radius: 10rem; } +.bg-transparent { + background-color: transparent !important; +} .rounded-circle { border-radius: 50% !important; } @@ -339,6 +342,20 @@ h1 { .m-auto { margin: auto !important; } +.gl-badge { + display: inline-flex; + align-items: center; + font-size: 0.75rem; + font-weight: 400; + line-height: 1rem; + padding-top: 0.25rem; + padding-bottom: 0.25rem; + padding-left: 0.5rem; + padding-right: 0.5rem; +} +.gl-button .gl-badge { + top: 0; +} .gl-form-input, .gl-form-input.form-control { background-color: #fff; @@ -836,6 +853,12 @@ input { .navbar-nav .badge.badge-pill:not(.merge-request-badge).todos-count { background-color: var(--blue-400, #428fdc); } +.title-container .canary-badge .badge, +.navbar-nav .canary-badge .badge { + font-size: 12px; + line-height: 16px; + padding: 0 0.5rem; +} @media (max-width: 575.98px) { .navbar-gitlab .container-fluid { font-size: 18px; @@ -1591,9 +1614,18 @@ svg.s16 { white-space: nowrap; width: 1px; } +.gl-bg-green-500 { + background-color: #108548; +} .gl-border-none\! { border-style: none !important; } +.gl-rounded-pill { + border-radius: 0.75rem; +} +.gl-text-white { + color: #fff; +} .gl-display-none { display: none; } @@ -1612,6 +1644,10 @@ svg.s16 { .gl-pr-2 { padding-right: 0.25rem; } +.gl-py-1 { + padding-top: 0.125rem; + padding-bottom: 0.125rem; +} .gl-ml-3 { margin-left: 0.5rem; } diff --git a/app/helpers/issues_helper.rb b/app/helpers/issues_helper.rb index 40e86b4623c..49f7d9aeef1 100644 --- a/app/helpers/issues_helper.rb +++ b/app/helpers/issues_helper.rb @@ -238,9 +238,10 @@ module IssuesHelper ) end - def group_issues_list_data(group, current_user, issues) + def group_issues_list_data(group, current_user, issues, projects) common_issues_list_data(group, current_user).merge( - has_any_issues: issues.to_a.any?.to_s + has_any_issues: issues.to_a.any?.to_s, + has_any_projects: any_projects?(projects).to_s ) end diff --git a/app/helpers/routing/pseudonymization_helper.rb b/app/helpers/routing/pseudonymization_helper.rb index e8202a4fb82..b73e49803ae 100644 --- a/app/helpers/routing/pseudonymization_helper.rb +++ b/app/helpers/routing/pseudonymization_helper.rb @@ -28,7 +28,7 @@ module Routing when 'groups' "/namespace:#{group.id}" when 'projects' - "/namespace:#{project.namespace.id}/project:#{project.id}" + "/namespace:#{project.namespace_id}/project:#{project.id}" when 'root' '' else @@ -44,7 +44,7 @@ module Routing masked_url = "#{request.protocol}#{request.host_with_port}" if request_params.has_key?(:project_id) - masked_url += "/namespace:#{project.namespace.id}/project:#{project.id}/-/#{namespace_type}" + masked_url += "/namespace:#{project.namespace_id}/project:#{project.id}/-/#{namespace_type}" end if request_params.has_key?(:id) diff --git a/app/helpers/tab_helper.rb b/app/helpers/tab_helper.rb index 05b7a53a38d..a6bb2f3b246 100644 --- a/app/helpers/tab_helper.rb +++ b/app/helpers/tab_helper.rb @@ -73,7 +73,6 @@ module TabHelper # :action - One or more action names to check (optional). # :path - A shorthand path, such as 'dashboard#index', to check (optional). # :html_options - Extra options to be passed to the list element (optional). - # :unless - Callable object to skip rendering the 'active' class on `li` element (optional). # block - An optional block that will become the contents of the returned # `li` element. # @@ -118,11 +117,6 @@ module TabHelper # nav_link(path: 'admin/appearances#show') { "Hello"} # # => '<li class="active">Hello</li>' # - # # Shorthand path + unless - # # Add `active` class when TreeController is requested, except the `index` action. - # nav_link(controller: 'tree', unless: -> { action_name?('index') }) { "Hello" } - # # => '<li class="active">Hello</li>' - # # # When `TreeController#index` is requested # # => '<li>Hello</li>' # @@ -151,8 +145,6 @@ module TabHelper end def active_nav_link?(options) - return false if options[:unless]&.call - controller = options.delete(:controller) action = options.delete(:action) diff --git a/app/models/ci/ci_database_record.rb b/app/models/ci/ci_database_record.rb index e2b832a28e7..e76b7a05b93 100644 --- a/app/models/ci/ci_database_record.rb +++ b/app/models/ci/ci_database_record.rb @@ -12,6 +12,13 @@ module Ci if Gitlab::Database.has_config?(:ci) connects_to database: { writing: :ci, reading: :ci } + + # TODO: Load Balancing messes with `CiDatabaseRecord` + # returning wrong connection. To be removed once merged: + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67773 + def self.connection + retrieve_connection + end end end end diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb index f35370c427f..2612001eb95 100644 --- a/app/services/projects/overwrite_project_service.rb +++ b/app/services/projects/overwrite_project_service.rb @@ -3,7 +3,7 @@ module Projects class OverwriteProjectService < BaseService def execute(source_project) - return unless source_project && source_project.namespace == @project.namespace + return unless source_project && source_project.namespace_id == @project.namespace_id start_time = ::Gitlab::Metrics::System.monotonic_time @@ -40,7 +40,7 @@ module Projects duration = ::Gitlab::Metrics::System.monotonic_time - start_time Gitlab::AppJsonLogger.info(class: self.class.name, - namespace_id: source_project.namespace.id, + namespace_id: source_project.namespace_id, project_id: source_project.id, duration_s: duration.to_f, error: exception.class.name) diff --git a/app/views/dashboard/_groups_head.html.haml b/app/views/dashboard/_groups_head.html.haml index b92f35c108c..7b1d25b9b43 100644 --- a/app/views/dashboard/_groups_head.html.haml +++ b/app/views/dashboard/_groups_head.html.haml @@ -6,13 +6,9 @@ = link_to _("New group"), new_group_path, class: "gl-button btn btn-confirm", data: { testid: "new-group-button" } .top-area - %ul.nav-links.mobile-separator.nav.nav-tabs - = nav_link(page: dashboard_groups_path) do - = link_to dashboard_groups_path, title: _("Your groups") do - Your groups - = nav_link(page: explore_groups_path) do - = link_to explore_groups_path, title: _("Explore public groups") do - Explore public groups + = gl_tabs_nav({ class: 'gl-flex-grow-1 gl-border-0' }) do + = gl_tab_link_to _("Your groups"), dashboard_groups_path + = gl_tab_link_to _("Explore public groups"), explore_groups_path .nav-controls = render 'shared/groups/search_form' = render 'shared/groups/dropdown' diff --git a/app/views/groups/_home_panel.html.haml b/app/views/groups/_home_panel.html.haml index 0352f366f5d..e530d9c60b6 100644 --- a/app/views/groups/_home_panel.html.haml +++ b/app/views/groups/_home_panel.html.haml @@ -5,17 +5,19 @@ .group-home-panel .row.mb-3 .home-panel-title-row.col-md-12.col-lg-6.d-flex - .avatar-container.rect-avatar.s64.home-panel-avatar.gl-mr-3.float-none + .avatar-container.rect-avatar.s64.home-panel-avatar.gl-flex-shrink-0.float-none{ class: 'gl-mr-3!' } = group_icon(@group, class: 'avatar avatar-tile s64', width: 64, height: 64, itemprop: 'logo') .d-flex.flex-column.flex-wrap.align-items-baseline .d-inline-flex.align-items-baseline - %h1.home-panel-title.gl-mt-3.gl-mb-2{ itemprop: 'name' } + %h1.home-panel-title.gl-mt-3.gl-mb-2.gl-ml-3{ itemprop: 'name' } = @group.name %span.visibility-icon.text-secondary.gl-ml-2.has-tooltip{ data: { container: 'body' }, title: visibility_icon_description(@group) } = visibility_level_icon(@group.visibility_level, options: {class: 'icon'}) - .home-panel-metadata.text-secondary - %span - = _("Group ID: %{group_id}") % { group_id: @group.id } + .home-panel-metadata.text-secondary.gl-font-base.gl-font-weight-normal.gl-line-height-normal + - if can?(current_user, :read_group, @group) + - button_class = "btn gl-button btn-sm btn-tertiary btn-default-tertiary home-panel-metadata" + - button_text = s_('GroupPage|Group ID: %{group_id}') % { group_id: @group.id } + = clipboard_button(title: s_('GroupPage|Copy group ID'), text: @group.id, hide_button_icon: true, button_text: button_text, class: button_class, qa_selector: 'group_id_content', itemprop: 'identifier') - if current_user %span.gl-ml-3 = render 'shared/members/access_request_links', source: @group diff --git a/app/views/groups/issues.html.haml b/app/views/groups/issues.html.haml index 1f746484b7d..0c6776a6038 100644 --- a/app/views/groups/issues.html.haml +++ b/app/views/groups/issues.html.haml @@ -6,7 +6,7 @@ = auto_discovery_link_tag(:atom, safe_params.merge(rss_url_options).to_h, title: "#{@group.name} issues") - if Feature.enabled?(:vue_issues_list, @group, default_enabled: :yaml) - .js-issues-list{ data: group_issues_list_data(@group, current_user, @issues) } + .js-issues-list{ data: group_issues_list_data(@group, current_user, @issues, @projects) } - if @can_bulk_update = render_if_exists 'shared/issuable/group_bulk_update_sidebar', group: @group, type: :issues - else diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index 6b4505ad1b3..1cb37e3d650 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -12,8 +12,7 @@ When a pipeline job is about to run, GitLab generates a unique token and injects You can use a GitLab CI/CD job token to authenticate with specific API endpoints: - Packages: - - [Package Registry](../../user/packages/package_registry/index.md#use-gitlab-cicd-to-build-packages). To push to the - Package Registry, you can use [deploy tokens](../../user/project/deploy_tokens/index.md). + - [Package Registry](../../user/packages/package_registry/index.md#use-gitlab-cicd-to-build-packages). - [Container Registry](../../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) (the `$CI_REGISTRY_PASSWORD` is `$CI_JOB_TOKEN`). - [Container Registry API](../../api/container_registry.md) diff --git a/doc/integration/vault.md b/doc/integration/vault.md index ebfa91c7516..5d2813a9f01 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# Vault Authentication with GitLab OpenID Connect +# Vault Authentication with GitLab OpenID Connect **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/22323) in GitLab 9.0 diff --git a/doc/user/admin_area/review_abuse_reports.md b/doc/user/admin_area/review_abuse_reports.md index 6494934c34d..6a8b48e7ba7 100644 --- a/doc/user/admin_area/review_abuse_reports.md +++ b/doc/user/admin_area/review_abuse_reports.md @@ -12,14 +12,14 @@ View and resolve abuse reports from GitLab users. GitLab administrators can view and [resolve](#resolving-abuse-reports) abuse reports in the Admin Area. -## Receiving notifications of abuse reports +## Receive notification of abuse reports by email -To receive notifications of new abuse reports by email, follow these steps: +To receive notifications of new abuse reports by email: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Reporting**. 1. Expand the **Abuse reports** section. -1. Provide an email address. +1. Provide an email address and select **Save changes**. The notification email address can also be set and retrieved [using the API](../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). diff --git a/doc/user/application_security/dast/dast_troubleshooting.md b/doc/user/application_security/dast/dast_troubleshooting.md index f771bc82d58..9969526c906 100644 --- a/doc/user/application_security/dast/dast_troubleshooting.md +++ b/doc/user/application_security/dast/dast_troubleshooting.md @@ -20,7 +20,7 @@ A DAST job has two executing processes: Enable the `DAST_DEBUG` CI/CD variable to debug scripts. This can help when troubleshooting the job, and outputs statements indicating what percentage of the scan is complete. -For details on using variables, see [Overriding the DAST template](index.md#customizing-the-dast-settings). +For details on using variables, see [Overriding the DAST template](index.md#customize-dast-settings). Debug mode of the ZAP server can be enabled using the `DAST_ZAP_LOG_CONFIGURATION` variable. The following table outlines examples of values that can be set and the effect that they have on the output that is logged. diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index cbd61747c62..5f7e6442ec0 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -483,6 +483,13 @@ When using `DAST_PATHS` and `DAST_PATHS_FILE`, note the following: To perform a [full scan](#full-scan) on the listed paths, use the `DAST_FULL_SCAN_ENABLED` CI/CD variable. +### List URLs scanned + +When DAST completes scanning, the merge request page states the number of URLs scanned. +Click **View details** to view the web console output which includes the list of scanned URLs. + + + ### View details of a vulnerability detected by DAST > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36332) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.1. @@ -515,15 +522,20 @@ To view details of vulnerabilities detected by DAST: | Links | Links to further details of the detected vulnerability. | | Solution | Details of a recommended solution to the vulnerability (optional). | -### Customizing the DAST settings +## Customize DAST settings + +You can customize the behavior of DAST using both CI/CD variables and command-line options. Use of CI/CD +variables overrides the values contained in the DAST template. + +### Customize DAST using CI/CD variables WARNING: Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/index.md#only--except) -is no longer supported. When overriding the template, you must use [`rules`](../../../ci/yaml/index.md#rules) instead. +is no longer supported. You must use [`rules`](../../../ci/yaml/index.md#rules) instead. The DAST settings can be changed through CI/CD variables by using the -[`variables`](../../../ci/yaml/index.md#variables) parameter in `.gitlab-ci.yml`. -These variables are documented in [available variables](#available-cicd-variables). +[`variables`](../../../ci/yaml/index.md#variables) parameter in `.gitlab-ci.yml`. For details of +all DAST CI/CD variables, read [Available CI/CD variables](#available-cicd-variables). For example: @@ -539,10 +551,10 @@ variables: Because the template is [evaluated before](../../../ci/yaml/index.md#include) the pipeline configuration, the last mention of the variable takes precedence. -#### Enabling and disabling rules +#### Enable or disable rules A complete list of the rules that DAST uses to scan for vulnerabilities can be -found in the [ZAP docs](https://www.zaproxy.org/docs/alerts/). +found in the [ZAP documentation](https://www.zaproxy.org/docs/alerts/). `DAST_EXCLUDE_RULES` disables the rules with the given IDs. @@ -559,7 +571,7 @@ can be found in [exclude_rules.yml](https://gitlab.com/gitlab-org/security-produ The lists for `DAST_EXCLUDE_RULES` and `DAST_ONLY_INCLUDE_RULES` **must** be enclosed in double quotes (`"`), otherwise they are interpreted as numeric values. -### Hide sensitive information +#### Hide sensitive information > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36332) in GitLab 13.1. @@ -573,7 +585,105 @@ authorization credentials. By default, the following headers are masked: Using the [`DAST_MASK_HTTP_HEADERS` CI/CD variable](#available-cicd-variables), you can list the headers whose values you want masked. For details on how to mask headers, see -[Customizing the DAST settings](#customizing-the-dast-settings). +[Customizing the DAST settings](#customize-dast-settings). + +#### Available CI/CD variables + +These CI/CD variables are specific to DAST. They can be used to customize the behavior of DAST to your requirements. + +| CI/CD variable | Type | Description | +|:-------------------------------------------------|:--------------|:------------------------------| +| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | +| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | +| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | +| `DAST_API_OPENAPI` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/290241) in GitLab 13.12 and replaced by `DAST_API_OPENAPI`. To be removed in GitLab 15.0. The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_AUTH_REPORT` <sup>2</sup> | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | +| `DAST_AUTH_EXCLUDE_URLS` <sup>2</sup> | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. | +| `DAST_AUTH_URL` <sup>1,2</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. Example: `https://login.example.com`. | +| `DAST_AUTH_VERIFICATION_LOGIN_FORM` <sup>2</sup> | boolean | Verifies successful authentication by checking for the lack of a login form once the login form has been submitted. | +| `DAST_AUTH_VERIFICATION_SELECTOR` <sup>2</sup> | selector | Verifies successful authentication by checking for presence of a selector once the login form has been submitted. Example: `css:.user-photo`. | +| `DAST_AUTH_VERIFICATION_URL` <sup>1,2</sup> | URL | A URL only accessible to logged in users that DAST can use to confirm successful authentication. If provided, DAST exits if it cannot access the URL. Example: `"http://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | +| `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | +| `DAST_BROWSER_PATH_TO_LOGIN_FORM` <sup>1,2</sup> | selector | Comma-separated list of selectors that will be clicked on prior to attempting to enter `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326633) in GitLab 14.1. | +| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | +| `DAST_EXCLUDE_URLS` <sup>1,2</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. Example, `http://example.com/sign-out`. | +| `DAST_FIRST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when clicked submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Not supported for API scans. Default: `false` | +| `DAST_FULL_SCAN_ENABLED` <sup>1</sup> | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | +| `DAST_HTML_REPORT` | string | The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_MARKDOWN_REPORT` | string | The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | +| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | +| `DAST_PASSWORD` <sup>1,2</sup> | string | The password to authenticate to in the website. Example: `P@55w0rd!` | +| `DAST_PASSWORD_FIELD` <sup>1,2</sup> | string | The selector of password field at the sign-in HTML form. Example: `id:password` | +| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | +| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | +| `DAST_REQUEST_HEADERS` <sup>1</sup> | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | +| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | +| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | +| `DAST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when clicked submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_TARGET_AVAILABILITY_TIMEOUT` <sup>1</sup> | number | Time limit in seconds to wait for target availability. | +| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_USERNAME` <sup>1,2</sup> | string | The username to authenticate to in the website. Example: `admin` | +| `DAST_USERNAME_FIELD` <sup>1,2</sup> | string | The selector of username field at the sign-in HTML form. Example: `name:username` | +| `DAST_XML_REPORT` | string | The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. The variable `DAST_API_OPENAPI` must be specified if this is omitted. | +| `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_ZAP_LOG_CONFIGURATION` | string | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `log4j.logger.org.parosproxy.paros.network.HttpSender=DEBUG;log4j.logger.com.crawljax=DEBUG` | +| `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | + +1. Available to an on-demand DAST scan. +1. Used for authentication. + +### Customize DAST using command-line options + +Not all DAST configuration is available via CI/CD variables. To find out all +possible options, run the following configuration. +Available command-line options are printed to the job log: + +```yaml +include: + template: DAST.gitlab-ci.yml + +dast: + script: + - /analyze --help +``` + +You must then overwrite the `script` command to pass in the appropriate +argument. For example, vulnerability definitions in alpha can be included with +`-a`. The following configuration includes those definitions: + +```yaml +include: + template: DAST.gitlab-ci.yml + +dast: + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - /analyze -a -t $DAST_WEBSITE +``` + +### Custom ZAProxy configuration + +The ZAProxy server contains many [useful configurable values](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_245801885). +Many key/values for `-config` remain undocumented, but there is an untested list of +[possible keys](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_244981023). +Note that these options are not supported by DAST, and may break the DAST scan +when used. An example of how to rewrite the Authorization header value with `TOKEN` follows: + +```yaml +include: + template: DAST.gitlab-ci.yml + +variables: + DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN" +``` ## Authentication @@ -747,59 +857,6 @@ dast: when: always ``` -## Available CI/CD variables - -These CI/CD variables are specific to DAST. They can be used to customize the behavior of DAST to your requirements. - -| CI/CD variable | Type | Description | -|:-------------------------------------------------|:--------------|:------------------------------| -| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | -| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | -| `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | -| `DAST_API_OPENAPI` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/290241) in GitLab 13.12 and replaced by `DAST_API_OPENAPI`. To be removed in GitLab 15.0. The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_AUTH_REPORT` <sup>2</sup> | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | -| `DAST_AUTH_EXCLUDE_URLS` <sup>2</sup> | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. | -| `DAST_AUTH_URL` <sup>1,2</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. Example: `https://login.example.com`. | -| `DAST_AUTH_VERIFICATION_LOGIN_FORM` <sup>2</sup> | boolean | Verifies successful authentication by checking for the lack of a login form once the login form has been submitted. | -| `DAST_AUTH_VERIFICATION_SELECTOR` <sup>2</sup> | selector | Verifies successful authentication by checking for presence of a selector once the login form has been submitted. Example: `css:.user-photo`. | -| `DAST_AUTH_VERIFICATION_URL` <sup>1,2</sup> | URL | A URL only accessible to logged in users that DAST can use to confirm successful authentication. If provided, DAST exits if it cannot access the URL. Example: `"http://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | -| `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | -| `DAST_BROWSER_PATH_TO_LOGIN_FORM` <sup>1,2</sup> | selector | Comma-separated list of selectors that will be clicked on prior to attempting to enter `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326633) in GitLab 14.1. | -| `DAST_DEBUG` <sup>1</sup> | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. The whole list **must** be enclosed in double quotes (`"`). Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | -| `DAST_EXCLUDE_URLS` <sup>1,2</sup> | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. Example, `http://example.com/sign-out`. | -| `DAST_FIRST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when clicked submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | -| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Not supported for API scans. Default: `false` | -| `DAST_FULL_SCAN_ENABLED` <sup>1</sup> | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | -| `DAST_HTML_REPORT` | string | The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MARKDOWN_REPORT` | string | The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | -| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | -| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. The whole list **must** be enclosed in double quotes (`"`). Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | -| `DAST_PASSWORD` <sup>1,2</sup> | string | The password to authenticate to in the website. Example: `P@55w0rd!` | -| `DAST_PASSWORD_FIELD` <sup>1,2</sup> | string | The selector of password field at the sign-in HTML form. Example: `id:password` | -| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | -| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | -| `DAST_REQUEST_HEADERS` <sup>1</sup> | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | -| `DAST_SPIDER_MINS` <sup>1</sup> | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | -| `DAST_SUBMIT_FIELD` <sup>2</sup> | string | The `id` or `name` of the element that when clicked submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | -| `DAST_TARGET_AVAILABILITY_TIMEOUT` <sup>1</sup> | number | Time limit in seconds to wait for target availability. | -| `DAST_USE_AJAX_SPIDER` <sup>1</sup> | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_USERNAME` <sup>1,2</sup> | string | The username to authenticate to in the website. Example: `admin` | -| `DAST_USERNAME_FIELD` <sup>1,2</sup> | string | The selector of username field at the sign-in HTML form. Example: `name:username` | -| `DAST_XML_REPORT` | string | The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. The variable `DAST_API_OPENAPI` must be specified if this is omitted. | -| `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_ZAP_LOG_CONFIGURATION` | string | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `log4j.logger.org.parosproxy.paros.network.HttpSender=DEBUG;log4j.logger.com.crawljax=DEBUG` | -| `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | - -1. Available to an on-demand DAST scan. -1. Used for authentication. - ### Selectors Selectors are used by CI/CD variables to specify the location of an element displayed on a page in a browser. @@ -848,51 +905,6 @@ When using selectors to locate specific fields we recommend you avoid searching - XPath searches as they are less performant than other selector searches. - Unscoped searches, such as those beginning with `css:*` and `xpath://*`. -### DAST command-line options - -Not all DAST configuration is available via CI/CD variables. To find out all -possible options, run the following configuration. -Available command-line options are printed to the job log: - -```yaml -include: - template: DAST.gitlab-ci.yml - -dast: - script: - - /analyze --help -``` - -You must then overwrite the `script` command to pass in the appropriate -argument. For example, vulnerability definitions in alpha can be included with -`-a`. The following configuration includes those definitions: - -```yaml -include: - template: DAST.gitlab-ci.yml - -dast: - script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} - - /analyze -a -t $DAST_WEBSITE -``` - -### Custom ZAProxy configuration - -The ZAProxy server contains many [useful configurable values](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_245801885). -Many key/values for `-config` remain undocumented, but there is an untested list of -[possible keys](https://gitlab.com/gitlab-org/gitlab/-/issues/36437#note_244981023). -Note that these options are not supported by DAST, and may break the DAST scan -when used. An example of how to rewrite the Authorization header value with `TOKEN` follows: - -```yaml -include: - template: DAST.gitlab-ci.yml - -variables: - DAST_ZAP_CLI_OPTIONS: "-config replacer.full_list(0).description=auth -config replacer.full_list(0).enabled=true -config replacer.full_list(0).matchtype=REQ_HEADER -config replacer.full_list(0).matchstr=Authorization -config replacer.full_list(0).regex=false -config replacer.full_list(0).replacement=TOKEN" -``` - ### Bleeding-edge vulnerability definitions ZAP first creates rules in the `alpha` class. After a testing period with @@ -1306,7 +1318,7 @@ If a scanner profile is linked to a security policy, a user cannot delete the pr page. See [Scan Execution Policies](../policies/index.md#scan-execution-policy-editor) for more information. -### Auditing +## Auditing > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1. @@ -1318,13 +1330,6 @@ and DAST site profiles are included in the [audit log](../../../administration/a The DAST tool outputs a report file in JSON format by default. However, this tool can also generate reports in Markdown, HTML, and XML. For more information, see the [schema for DAST reports](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dast-report-format.json). -### List of URLs scanned - -When DAST completes scanning, the merge request page states the number of URLs scanned. -Click **View details** to view the web console output which includes the list of scanned URLs. - - - ### JSON WARNING: diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb index 42bbfb32d0b..45c7dc295b1 100644 --- a/lib/gitlab/issuable_sorter.rb +++ b/lib/gitlab/issuable_sorter.rb @@ -7,7 +7,7 @@ module Gitlab grouped_items = issuables.group_by do |issuable| if issuable.project.id == project.id :project_ref - elsif issuable.project.namespace.id == project.namespace.id + elsif issuable.project.namespace_id == project.namespace_id :namespace_ref else :full_ref diff --git a/locale/gitlab.pot b/locale/gitlab.pot index c4149d60508..63f50708d23 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3710,6 +3710,9 @@ msgstr "" msgid "An error occurred while loading merge requests." msgstr "" +msgid "An error occurred while loading projects." +msgstr "" + msgid "An error occurred while loading the Needs tab." msgstr "" @@ -15922,9 +15925,6 @@ msgstr "" msgid "Group ID" msgstr "" -msgid "Group ID: %{group_id}" -msgstr "" - msgid "Group Owner must have signed in with SAML before enabling Group Managed Accounts" msgstr "" @@ -16099,6 +16099,12 @@ msgstr "" msgid "GroupImport|Unable to process group import file" msgstr "" +msgid "GroupPage|Copy group ID" +msgstr "" + +msgid "GroupPage|Group ID: %{group_id}" +msgstr "" + msgid "GroupRoadmap|%{dateWord} – No end date" msgstr "" @@ -22705,6 +22711,9 @@ msgstr "" msgid "New issue" msgstr "" +msgid "New issue in %{project}" +msgstr "" + msgid "New issue title" msgstr "" @@ -30553,6 +30562,9 @@ msgstr "" msgid "Select project to choose zone" msgstr "" +msgid "Select project to create issue" +msgstr "" + msgid "Select projects" msgstr "" diff --git a/spec/frontend/fixtures/startup_css.rb b/spec/frontend/fixtures/startup_css.rb index 1bd99f5cd7f..067753207d2 100644 --- a/spec/frontend/fixtures/startup_css.rb +++ b/spec/frontend/fixtures/startup_css.rb @@ -18,6 +18,10 @@ RSpec.describe 'Startup CSS fixtures', type: :controller do let(:project) { create(:project, :public, :repository, description: 'Code and stuff', creator: user) } before do + # We want vNext badge to be included and com/canary don't remove/hide any other elements. + # This is why we're turning com and canary on by default for now. + allow(Gitlab).to receive(:com?).and_return(true) + allow(Gitlab).to receive(:canary?).and_return(true) sign_in(user) end diff --git a/spec/frontend/issues_list/components/issues_list_app_spec.js b/spec/frontend/issues_list/components/issues_list_app_spec.js index 8d79a5eed35..3a1f732a4e9 100644 --- a/spec/frontend/issues_list/components/issues_list_app_spec.js +++ b/spec/frontend/issues_list/components/issues_list_app_spec.js @@ -24,6 +24,7 @@ import IssuableByEmail from '~/issuable/components/issuable_by_email.vue'; import IssuableList from '~/issuable_list/components/issuable_list_root.vue'; import { IssuableListTabs, IssuableStates } from '~/issuable_list/constants'; import IssuesListApp from '~/issues_list/components/issues_list_app.vue'; +import NewIssueDropdown from '~/issues_list/components/new_issue_dropdown.vue'; import { CREATED_DESC, DUE_DATE_OVERDUE, @@ -65,6 +66,7 @@ describe('IssuesListApp component', () => { exportCsvPath: 'export/csv/path', fullPath: 'path/to/project', hasAnyIssues: true, + hasAnyProjects: true, hasBlockedIssuesFeature: true, hasIssueWeightsFeature: true, hasIterationsFeature: true, @@ -93,6 +95,7 @@ describe('IssuesListApp component', () => { const findGlEmptyState = () => wrapper.findComponent(GlEmptyState); const findGlLink = () => wrapper.findComponent(GlLink); const findIssuableList = () => wrapper.findComponent(IssuableList); + const findNewIssueDropdown = () => wrapper.findComponent(NewIssueDropdown); const mountComponent = ({ provide = {}, @@ -190,10 +193,7 @@ describe('IssuesListApp component', () => { beforeEach(() => { setWindowLocation(search); - wrapper = mountComponent({ - provide: { isSignedIn: true }, - mountFn: mount, - }); + wrapper = mountComponent({ provide: { isSignedIn: true }, mountFn: mount }); jest.runOnlyPendingTimers(); }); @@ -208,7 +208,7 @@ describe('IssuesListApp component', () => { describe('when user is not signed in', () => { it('does not render', () => { - wrapper = mountComponent({ provide: { isSignedIn: false } }); + wrapper = mountComponent({ provide: { isSignedIn: false }, mountFn: mount }); expect(findCsvImportExportButtons().exists()).toBe(false); }); @@ -216,7 +216,7 @@ describe('IssuesListApp component', () => { describe('when in a group context', () => { it('does not render', () => { - wrapper = mountComponent({ provide: { isProject: false } }); + wrapper = mountComponent({ provide: { isProject: false }, mountFn: mount }); expect(findCsvImportExportButtons().exists()).toBe(false); }); @@ -231,7 +231,7 @@ describe('IssuesListApp component', () => { }); it('does not render when user does not have permissions', () => { - wrapper = mountComponent({ provide: { canBulkUpdate: false } }); + wrapper = mountComponent({ provide: { canBulkUpdate: false }, mountFn: mount }); expect(findGlButtons().filter((button) => button.text() === 'Edit issues')).toHaveLength(0); }); @@ -258,11 +258,25 @@ describe('IssuesListApp component', () => { }); it('does not render when user does not have permissions', () => { - wrapper = mountComponent({ provide: { showNewIssueLink: false } }); + wrapper = mountComponent({ provide: { showNewIssueLink: false }, mountFn: mount }); expect(findGlButtons().filter((button) => button.text() === 'New issue')).toHaveLength(0); }); }); + + describe('new issue split dropdown', () => { + it('does not render in a project context', () => { + wrapper = mountComponent({ provide: { isProject: true }, mountFn: mount }); + + expect(findNewIssueDropdown().exists()).toBe(false); + }); + + it('renders in a group context', () => { + wrapper = mountComponent({ provide: { isProject: false }, mountFn: mount }); + + expect(findNewIssueDropdown().exists()).toBe(true); + }); + }); }); describe('initial url params', () => { diff --git a/spec/frontend/issues_list/components/new_issue_dropdown_spec.js b/spec/frontend/issues_list/components/new_issue_dropdown_spec.js new file mode 100644 index 00000000000..1fcaa99cf5a --- /dev/null +++ b/spec/frontend/issues_list/components/new_issue_dropdown_spec.js @@ -0,0 +1,131 @@ +import { GlDropdown, GlDropdownItem, GlSearchBoxByType } from '@gitlab/ui'; +import { createLocalVue, mount, shallowMount } from '@vue/test-utils'; +import VueApollo from 'vue-apollo'; +import createMockApollo from 'helpers/mock_apollo_helper'; +import NewIssueDropdown from '~/issues_list/components/new_issue_dropdown.vue'; +import searchProjectsQuery from '~/issues_list/queries/search_projects.query.graphql'; +import { DASH_SCOPE, joinPaths } from '~/lib/utils/url_utility'; +import { + emptySearchProjectsQueryResponse, + project1, + project2, + searchProjectsQueryResponse, +} from '../mock_data'; + +describe('NewIssueDropdown component', () => { + let wrapper; + + const localVue = createLocalVue(); + localVue.use(VueApollo); + + const mountComponent = ({ + search = '', + queryResponse = searchProjectsQueryResponse, + mountFn = shallowMount, + } = {}) => { + const requestHandlers = [[searchProjectsQuery, jest.fn().mockResolvedValue(queryResponse)]]; + const apolloProvider = createMockApollo(requestHandlers); + + return mountFn(NewIssueDropdown, { + localVue, + apolloProvider, + provide: { + fullPath: 'mushroom-kingdom', + }, + data() { + return { search }; + }, + }); + }; + + const findDropdown = () => wrapper.findComponent(GlDropdown); + const findInput = () => wrapper.findComponent(GlSearchBoxByType); + const showDropdown = async () => { + findDropdown().vm.$emit('shown'); + await wrapper.vm.$apollo.queries.projects.refetch(); + jest.runOnlyPendingTimers(); + }; + + afterEach(() => { + wrapper.destroy(); + }); + + it('renders a split dropdown', () => { + wrapper = mountComponent(); + + expect(findDropdown().props('split')).toBe(true); + }); + + it('renders a label for the dropdown toggle button', () => { + wrapper = mountComponent(); + + expect(findDropdown().attributes('toggle-text')).toBe(NewIssueDropdown.i18n.toggleButtonLabel); + }); + + it('focuses on input when dropdown is shown', async () => { + wrapper = mountComponent({ mountFn: mount }); + + const inputSpy = jest.spyOn(findInput().vm, 'focusInput'); + + await showDropdown(); + + expect(inputSpy).toHaveBeenCalledTimes(1); + }); + + it('renders expected dropdown items', async () => { + wrapper = mountComponent({ mountFn: mount }); + + await showDropdown(); + + const listItems = wrapper.findAll('li'); + + expect(listItems.at(0).text()).toBe(project1.nameWithNamespace); + expect(listItems.at(1).text()).toBe(project2.nameWithNamespace); + }); + + it('renders `No matches found` when there are no matches', async () => { + wrapper = mountComponent({ + search: 'no matches', + queryResponse: emptySearchProjectsQueryResponse, + mountFn: mount, + }); + + await showDropdown(); + + expect(wrapper.find('li').text()).toBe(NewIssueDropdown.i18n.noMatchesFound); + }); + + describe('when no project is selected', () => { + beforeEach(() => { + wrapper = mountComponent(); + }); + + it('dropdown button is not a link', () => { + expect(findDropdown().attributes('split-href')).toBeUndefined(); + }); + + it('displays default text on the dropdown button', () => { + expect(findDropdown().props('text')).toBe(NewIssueDropdown.i18n.defaultDropdownText); + }); + }); + + describe('when a project is selected', () => { + beforeEach(async () => { + wrapper = mountComponent({ mountFn: mount }); + + await showDropdown(); + + wrapper.findComponent(GlDropdownItem).vm.$emit('click', project1); + }); + + it('dropdown button is a link', () => { + const href = joinPaths(project1.webUrl, DASH_SCOPE, 'issues/new'); + + expect(findDropdown().attributes('split-href')).toBe(href); + }); + + it('displays project name on the dropdown button', () => { + expect(findDropdown().props('text')).toBe(`New issue in ${project1.name}`); + }); + }); +}); diff --git a/spec/frontend/issues_list/mock_data.js b/spec/frontend/issues_list/mock_data.js index 720f9cac986..3be256d8094 100644 --- a/spec/frontend/issues_list/mock_data.js +++ b/spec/frontend/issues_list/mock_data.js @@ -221,3 +221,37 @@ export const urlParamsWithSpecialValues = { epic_id: 'None', weight: 'None', }; + +export const project1 = { + id: 'gid://gitlab/Group/26', + name: 'Super Mario Project', + nameWithNamespace: 'Mushroom Kingdom / Super Mario Project', + webUrl: 'https://127.0.0.1:3000/mushroom-kingdom/super-mario-project', +}; + +export const project2 = { + id: 'gid://gitlab/Group/59', + name: 'Mario Kart Project', + nameWithNamespace: 'Mushroom Kingdom / Mario Kart Project', + webUrl: 'https://127.0.0.1:3000/mushroom-kingdom/mario-kart-project', +}; + +export const searchProjectsQueryResponse = { + data: { + group: { + projects: { + nodes: [project1, project2], + }, + }, + }, +}; + +export const emptySearchProjectsQueryResponse = { + data: { + group: { + projects: { + nodes: [], + }, + }, + }, +}; diff --git a/spec/helpers/issues_helper_spec.rb b/spec/helpers/issues_helper_spec.rb index f5f26d306fb..850051c7875 100644 --- a/spec/helpers/issues_helper_spec.rb +++ b/spec/helpers/issues_helper_spec.rb @@ -354,6 +354,7 @@ RSpec.describe IssuesHelper do let(:group) { create(:group) } let(:current_user) { double.as_null_object } let(:issues) { [] } + let(:projects) { [] } it 'returns expected result' do allow(helper).to receive(:current_user).and_return(current_user) @@ -367,13 +368,14 @@ RSpec.describe IssuesHelper do empty_state_svg_path: '#', full_path: group.full_path, has_any_issues: issues.to_a.any?.to_s, + has_any_projects: any_projects?(projects).to_s, is_signed_in: current_user.present?.to_s, jira_integration_path: help_page_url('integration/jira/issues', anchor: 'view-jira-issues'), rss_path: '#', sign_in_path: new_user_session_path } - expect(helper.group_issues_list_data(group, current_user, issues)).to include(expected) + expect(helper.group_issues_list_data(group, current_user, issues, projects)).to include(expected) end end diff --git a/spec/helpers/routing/pseudonymization_helper_spec.rb b/spec/helpers/routing/pseudonymization_helper_spec.rb index 68c1ce5b625..a28a86d1f53 100644 --- a/spec/helpers/routing/pseudonymization_helper_spec.rb +++ b/spec/helpers/routing/pseudonymization_helper_spec.rb @@ -6,6 +6,7 @@ RSpec.describe ::Routing::PseudonymizationHelper do let_it_be(:group) { create(:group) } let_it_be(:subgroup) { create(:group, parent: group) } let_it_be(:project) { create(:project, group: group) } + let_it_be(:subproject) { create(:project, group: subgroup) } let_it_be(:issue) { create(:issue, project: project) } let(:merge_request) { create(:merge_request, source_project: project) } @@ -56,16 +57,16 @@ RSpec.describe ::Routing::PseudonymizationHelper do end context 'with controller for groups with subgroups and project' do - let(:masked_url) { "http://test.host/namespace:#{subgroup.id}/project:#{project.id}"} + let(:masked_url) { "http://test.host/namespace:#{subgroup.id}/project:#{subproject.id}"} before do allow(helper).to receive(:group).and_return(subgroup) - allow(helper.project).to receive(:namespace).and_return(subgroup) + allow(helper).to receive(:project).and_return(subproject) allow(Rails.application.routes).to receive(:recognize_path).and_return({ controller: 'projects', action: 'show', namespace_id: subgroup.name, - id: project.name + id: subproject.name }) end diff --git a/spec/lib/gitlab/database/schema_migrations/context_spec.rb b/spec/lib/gitlab/database/schema_migrations/context_spec.rb index c9fdcdb079c..0323fa22b78 100644 --- a/spec/lib/gitlab/database/schema_migrations/context_spec.rb +++ b/spec/lib/gitlab/database/schema_migrations/context_spec.rb @@ -24,16 +24,6 @@ RSpec.describe Gitlab::Database::SchemaMigrations::Context do end context 'multiple databases', :reestablished_active_record_base do - let(:connection_class) do - Class.new(::ApplicationRecord) do - self.abstract_class = true - - def self.name - 'Gitlab::Database::SchemaMigrations::Context::TestConnection' - end - end - end - before do connection_class.establish_connection( ActiveRecord::Base @@ -44,10 +34,6 @@ RSpec.describe Gitlab::Database::SchemaMigrations::Context do ) end - after do - connection_class.remove_connection - end - context 'when `schema_migrations_path` is configured as string' do let(:configuration_overrides) do { "schema_migrations_path" => "db/ci_schema_migrations" } diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index b31881e3082..8f706fdebc9 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -17,32 +17,9 @@ RSpec.configure do |config| delete_from_all_tables!(except: ['work_item_types']) # Postgres maximum number of columns in a table is 1600 (https://github.com/postgres/postgres/blob/de41869b64d57160f58852eab20a27f248188135/src/include/access/htup_details.h#L23-L47). - # And since: - # "The DROP COLUMN form does not physically remove the column, but simply makes - # it invisible to SQL operations. Subsequent insert and update operations in the - # table will store a null value for the column. Thus, dropping a column is quick - # but it will not immediately reduce the on-disk size of your table, as the space - # occupied by the dropped column is not reclaimed. - # The space will be reclaimed over time as existing rows are updated." - # according to https://www.postgresql.org/docs/current/sql-altertable.html. # We drop and recreate the database if any table has more than 1200 columns, just to be safe. - max_allowed_columns = 1200 - tables_with_more_than_allowed_columns = - ApplicationRecord.connection.execute("SELECT attrelid::regclass::text AS table, COUNT(*) AS column_count FROM pg_attribute GROUP BY attrelid HAVING COUNT(*) > #{max_allowed_columns}") - - if tables_with_more_than_allowed_columns.any? - tables_with_more_than_allowed_columns.each do |result| - puts "The #{result['table']} table has #{result['column_count']} columns." - end - puts "Recreating the database" - start = Gitlab::Metrics::System.monotonic_time - - ActiveRecord::Tasks::DatabaseTasks.drop_current - ActiveRecord::Tasks::DatabaseTasks.create_current - ActiveRecord::Tasks::DatabaseTasks.load_schema_current - ActiveRecord::Tasks::DatabaseTasks.migrate - - puts "Database re-creation done in #{Gitlab::Metrics::System.monotonic_time - start}" + if any_connection_class_with_more_than_allowed_columns? + recreate_all_databases! end end diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 940ff2751d3..316d645f99f 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -2,7 +2,7 @@ module DbCleaner def all_connection_classes - ::ActiveRecord::Base.connection_handler.connection_pool_names.map(&:constantize) + ::BeforeAllAdapter.all_connection_classes end def delete_from_all_tables!(except: []) @@ -20,6 +20,79 @@ module DbCleaner DatabaseCleaner[:active_record, { connection: connection_class }] end end + + def any_connection_class_with_more_than_allowed_columns? + all_connection_classes.any? do |connection_class| + more_than_allowed_columns?(connection_class) + end + end + + def more_than_allowed_columns?(connection_class) + # Postgres maximum number of columns in a table is 1600 (https://github.com/postgres/postgres/blob/de41869b64d57160f58852eab20a27f248188135/src/include/access/htup_details.h#L23-L47). + # And since: + # "The DROP COLUMN form does not physically remove the column, but simply makes + # it invisible to SQL operations. Subsequent insert and update operations in the + # table will store a null value for the column. Thus, dropping a column is quick + # but it will not immediately reduce the on-disk size of your table, as the space + # occupied by the dropped column is not reclaimed. + # The space will be reclaimed over time as existing rows are updated." + # according to https://www.postgresql.org/docs/current/sql-altertable.html. + # We drop and recreate the database if any table has more than 1200 columns, just to be safe. + max_allowed_columns = 1200 + tables_with_more_than_allowed_columns = connection_class.connection.execute(<<-SQL) + SELECT attrelid::regclass::text AS table, COUNT(*) AS column_count + FROM pg_attribute + GROUP BY attrelid + HAVING COUNT(*) > #{max_allowed_columns} + SQL + + tables_with_more_than_allowed_columns.each do |result| + puts "The #{result['table']} (#{connection_class.connection_db_config.name}) table has #{result['column_count']} columns." + end + + tables_with_more_than_allowed_columns.any? + end + + def recreate_all_databases! + start = Gitlab::Metrics::System.monotonic_time + + puts "Recreating the database" + + force_disconnect_all_connections! + + ActiveRecord::Tasks::DatabaseTasks.drop_current + ActiveRecord::Tasks::DatabaseTasks.create_current + ActiveRecord::Tasks::DatabaseTasks.load_schema_current + + # Migrate each database individually + with_reestablished_active_record_base do + all_connection_classes.each do |connection_class| + ActiveRecord::Base.establish_connection(connection_class.connection_db_config) + + ActiveRecord::Tasks::DatabaseTasks.migrate + end + end + + puts "Databases re-creation done in #{Gitlab::Metrics::System.monotonic_time - start}" + end + + def force_disconnect_all_connections! + all_connection_classes.each do |connection_class| + # We use `connection_pool` to avoid going through + # Load Balancer since it does retry ops + pool = connection_class.connection_pool + + # Force disconnect https://www.cybertec-postgresql.com/en/terminating-database-connections-in-postgresql/ + pool.connection.execute(<<-SQL) + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = #{pool.connection.quote(pool.db_config.database)} + AND pid != pg_backend_pid(); + SQL + + connection_class.connection_pool.disconnect! + end + end end DbCleaner.prepend_mod_with('DbCleaner') |