Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-11-30 18:09:21 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2020-11-30 18:09:21 +0300
commit56eafa995d0bbda39bc24cd07537286bf36a4dd9 (patch)
treeb172c7a9b29e75851d9c20b07147efcebd768be1
parentf66bb12f3879bf86387157af1e614c5e0e93e561 (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--.gitlab/ci/reports.gitlab-ci.yml4
-rw-r--r--app/assets/javascripts/boards/components/board_configuration_options.vue2
-rw-r--r--app/assets/javascripts/boards/components/board_form.vue2
-rw-r--r--app/helpers/application_helper.rb8
-rw-r--r--app/helpers/issuables_helper.rb49
-rw-r--r--app/models/bulk_imports/entity.rb6
-rw-r--r--app/models/bulk_imports/failure.rb13
-rw-r--r--app/views/import/manifest/_form.html.haml4
-rw-r--r--app/views/jira_connect/subscriptions/index.html.haml2
-rw-r--r--app/views/layouts/jira_connect.html.haml3
-rw-r--r--app/views/projects/merge_requests/_mr_title.html.haml6
-rw-r--r--app/views/shared/issuable/_close_reopen_draft_report_toggle.html.haml2
-rw-r--r--app/views/shared/issue_type/_details_header.html.haml2
-rw-r--r--changelogs/unreleased/231189-manifest-dir-buttons.yml5
-rw-r--r--changelogs/unreleased/267993-enable-by-default-true-in-api-helper-usage_data_-event-feature.yml5
-rw-r--r--changelogs/unreleased/273777-fix-css-not-loading-on-jira-connect-app.yml5
-rw-r--r--changelogs/unreleased/276897-add-external-issue-links.yml5
-rw-r--r--changelogs/unreleased/georgekoltsov-bulk-import-failures.yml6
-rw-r--r--config/feature_flags/development/usage_data_a_compliance_audit_events_api.yml2
-rw-r--r--config/feature_flags/development/usage_data_g_compliance_dashboard.yml8
-rw-r--r--db/migrate/20201112132808_create_bulk_import_failures.rb39
-rw-r--r--db/migrate/20201124075951_create_vulnerability_external_links.rb42
-rw-r--r--db/schema_migrations/202011121328081
-rw-r--r--db/schema_migrations/202011240759511
-rw-r--r--db/structure.sql77
-rw-r--r--doc/administration/instance_limits.md16
-rw-r--r--doc/administration/reference_architectures/10k_users.md75
-rw-r--r--doc/administration/reference_architectures/25k_users.md75
-rw-r--r--doc/administration/reference_architectures/2k_users.md40
-rw-r--r--doc/administration/reference_architectures/3k_users.md56
-rw-r--r--doc/administration/reference_architectures/50k_users.md75
-rw-r--r--doc/administration/reference_architectures/5k_users.md2
-rw-r--r--doc/api/graphql/reference/gitlab_schema.graphql41
-rw-r--r--doc/api/graphql/reference/gitlab_schema.json143
-rw-r--r--doc/api/graphql/reference/index.md10
-rw-r--r--doc/development/fe_guide/graphql.md94
-rw-r--r--doc/development/product_analytics/snowplow.md2
-rw-r--r--lib/api/helpers.rb5
-rw-r--r--lib/bulk_imports/groups/pipelines/group_pipeline.rb2
-rw-r--r--lib/bulk_imports/pipeline.rb14
-rw-r--r--lib/bulk_imports/pipeline/runner.rb95
-rw-r--r--locale/gitlab.pot3
-rw-r--r--package.json4
-rw-r--r--spec/factories/bulk_import/failures.rb14
-rw-r--r--spec/frontend/fixtures/issues.rb6
-rw-r--r--spec/lib/bulk_imports/importers/group_importer_spec.rb14
-rw-r--r--spec/lib/bulk_imports/pipeline/runner_spec.rb169
-rw-r--r--spec/lib/bulk_imports/pipeline_spec.rb6
-rw-r--r--spec/models/bulk_imports/failure_spec.rb17
-rw-r--r--yarn.lock18
50 files changed, 1143 insertions, 152 deletions
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index 565ed93967c..85aec070557 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -145,6 +145,10 @@ dependency_scanning:
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code
+ # Post-processing: This will be an after_script once this job will use the Dependency Scanning CI template
+ - apk add jq
+ # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390
+ - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json
artifacts:
paths:
- gl-dependency-scanning-report.json # GitLab-specific
diff --git a/app/assets/javascripts/boards/components/board_configuration_options.vue b/app/assets/javascripts/boards/components/board_configuration_options.vue
index 754b00b54e0..99d1e4a2611 100644
--- a/app/assets/javascripts/boards/components/board_configuration_options.vue
+++ b/app/assets/javascripts/boards/components/board_configuration_options.vue
@@ -42,7 +42,7 @@ export default {
</script>
<template>
- <div class="append-bottom-20">
+ <div class="gl-mb-5">
<label class="label-bold gl-font-lg" for="board-new-name">
{{ __('List options') }}
</label>
diff --git a/app/assets/javascripts/boards/components/board_form.vue b/app/assets/javascripts/boards/components/board_form.vue
index e4ef3600ff9..d464379400d 100644
--- a/app/assets/javascripts/boards/components/board_form.vue
+++ b/app/assets/javascripts/boards/components/board_form.vue
@@ -195,7 +195,7 @@ export default {
<template #body>
<p v-if="isDeleteForm">{{ __('Are you sure you want to delete this board?') }}</p>
<form v-else class="js-board-config-modal" @submit.prevent>
- <div v-if="!readonly" class="append-bottom-20">
+ <div v-if="!readonly" class="gl-mb-5">
<label class="label-bold gl-font-lg" for="board-new-name">{{ __('Title') }}</label>
<input
id="board-new-name"
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 2a6b00c0bd8..512ba7e2a66 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -361,9 +361,13 @@ module ApplicationHelper
}
end
- def add_page_specific_style(path)
+ def add_page_specific_style(path, defer: true)
content_for :page_specific_styles do
- stylesheet_link_tag_defer path
+ if defer
+ stylesheet_link_tag_defer path
+ else
+ stylesheet_link_tag path
+ end
end
end
diff --git a/app/helpers/issuables_helper.rb b/app/helpers/issuables_helper.rb
index 229239f53a5..1f289265916 100644
--- a/app/helpers/issuables_helper.rb
+++ b/app/helpers/issuables_helper.rb
@@ -61,16 +61,6 @@ module IssuablesHelper
end
end
- def issuable_json_path(issuable)
- project = issuable.project
-
- if issuable.is_a?(MergeRequest)
- project_merge_request_path(project, issuable.iid, :json)
- else
- project_issue_path(project, issuable.iid, :json)
- end
- end
-
def serialize_issuable(issuable, opts = {})
serializer_klass = case issuable
when Issue
@@ -174,18 +164,7 @@ module IssuablesHelper
h(title || default_label)
end
- def to_url_reference(issuable)
- case issuable
- when Issue
- link_to issuable.to_reference, issue_url(issuable)
- when MergeRequest
- link_to issuable.to_reference, merge_request_url(issuable)
- else
- issuable.to_reference
- end
- end
-
- def issuable_meta(issuable, project, text)
+ def issuable_meta(issuable, project)
output = []
output << "Opened #{time_ago_with_tooltip(issuable.created_at)} by ".html_safe
@@ -340,10 +319,6 @@ module IssuablesHelper
issuable_path(issuable, close_reopen_params(issuable, :reopen))
end
- def close_reopen_issuable_path(issuable, should_inverse = false)
- issuable.closed? ^ should_inverse ? reopen_issuable_path(issuable) : close_issuable_path(issuable)
- end
-
def toggle_draft_issuable_path(issuable)
wip_event = issuable.work_in_progress? ? 'unwip' : 'wip'
@@ -354,28 +329,6 @@ module IssuablesHelper
polymorphic_path(issuable, *options)
end
- def issuable_url(issuable, *options)
- case issuable
- when Issue
- issue_url(issuable, *options)
- when MergeRequest
- merge_request_url(issuable, *options)
- end
- end
-
- def issuable_button_visibility(issuable, closed)
- return 'hidden' if issuable_button_hidden?(issuable, closed)
- end
-
- def issuable_button_hidden?(issuable, closed)
- case issuable
- when Issue
- issue_button_hidden?(issuable, closed)
- when MergeRequest
- merge_request_button_hidden?(issuable, closed)
- end
- end
-
def issuable_author_is_current_user(issuable)
issuable.author == current_user
end
diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb
index f0f4d3ef339..a4d0b7485ba 100644
--- a/app/models/bulk_imports/entity.rb
+++ b/app/models/bulk_imports/entity.rb
@@ -30,6 +30,11 @@ class BulkImports::Entity < ApplicationRecord
class_name: 'BulkImports::Tracker',
foreign_key: :bulk_import_entity_id
+ has_many :failures,
+ class_name: 'BulkImports::Failure',
+ inverse_of: :entity,
+ foreign_key: :bulk_import_entity_id
+
validates :project, absence: true, if: :group
validates :group, absence: true, if: :project
validates :source_type, :source_full_path, :destination_name,
@@ -52,6 +57,7 @@ class BulkImports::Entity < ApplicationRecord
event :finish do
transition started: :finished
+ transition failed: :failed
end
event :fail_op do
diff --git a/app/models/bulk_imports/failure.rb b/app/models/bulk_imports/failure.rb
new file mode 100644
index 00000000000..a6f7582c3b0
--- /dev/null
+++ b/app/models/bulk_imports/failure.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class BulkImports::Failure < ApplicationRecord
+ self.table_name = 'bulk_import_failures'
+
+ belongs_to :entity,
+ class_name: 'BulkImports::Entity',
+ foreign_key: :bulk_import_entity_id,
+ inverse_of: :failures,
+ optional: false
+
+ validates :entity, presence: true
+end
diff --git a/app/views/import/manifest/_form.html.haml b/app/views/import/manifest/_form.html.haml
index 2ee964974c3..1a3b945cfe5 100644
--- a/app/views/import/manifest/_form.html.haml
+++ b/app/views/import/manifest/_form.html.haml
@@ -19,5 +19,5 @@
= link_to sprite_icon('question-o'), help_page_path('user/project/import/manifest')
.gl-mb-3
- = submit_tag _('List available repositories'), class: 'btn btn-success'
- = link_to _('Cancel'), new_project_path, class: 'btn btn-cancel'
+ = submit_tag _('List available repositories'), class: 'gl-button btn btn-success'
+ = link_to _('Cancel'), new_project_path, class: 'gl-button btn btn-default btn-cancel'
diff --git a/app/views/jira_connect/subscriptions/index.html.haml b/app/views/jira_connect/subscriptions/index.html.haml
index 355ffabd7ec..7bbb3c31801 100644
--- a/app/views/jira_connect/subscriptions/index.html.haml
+++ b/app/views/jira_connect/subscriptions/index.html.haml
@@ -63,4 +63,4 @@
= webpack_bundle_tag 'jira_connect_app'
= page_specific_javascript_tag('jira_connect.js')
-- add_page_specific_style 'page_bundles/jira_connect'
+- add_page_specific_style 'page_bundles/jira_connect', defer: false
diff --git a/app/views/layouts/jira_connect.html.haml b/app/views/layouts/jira_connect.html.haml
index 17f6e9af61a..8e1549f5f59 100644
--- a/app/views/layouts/jira_connect.html.haml
+++ b/app/views/layouts/jira_connect.html.haml
@@ -5,9 +5,10 @@
GitLab
= stylesheet_link_tag 'https://unpkg.com/@atlaskit/css-reset@3.0.6/dist/bundle.css'
= stylesheet_link_tag 'https://unpkg.com/@atlaskit/reduced-ui-pack@10.5.5/dist/bundle.css'
+ = yield :page_specific_styles
+
= javascript_include_tag 'https://connect-cdn.atl-paas.net/all.js'
= javascript_include_tag 'https://unpkg.com/jquery@3.3.1/dist/jquery.min.js'
- = yield :page_specific_styles
= yield :head
%body
.ac-content
diff --git a/app/views/projects/merge_requests/_mr_title.html.haml b/app/views/projects/merge_requests/_mr_title.html.haml
index f52a9ed9a4a..6c896720ec1 100644
--- a/app/views/projects/merge_requests/_mr_title.html.haml
+++ b/app/views/projects/merge_requests/_mr_title.html.haml
@@ -2,7 +2,7 @@
- can_update_merge_request = can?(current_user, :update_merge_request, @merge_request)
- can_reopen_merge_request = can?(current_user, :reopen_merge_request, @merge_request)
- state_human_name, state_icon_name = state_name_with_icon(@merge_request)
-- are_close_and_open_buttons_hidden = issuable_button_hidden?(@merge_request, true) && issuable_button_hidden?(@merge_request, false)
+- are_close_and_open_buttons_hidden = merge_request_button_hidden?(@merge_request, true) && merge_request_button_hidden?(@merge_request, false)
- if @merge_request.closed_without_fork?
.gl-alert.gl-alert-danger.gl-mb-5
@@ -19,7 +19,7 @@
.issuable-meta
#js-issuable-header-warnings
- = issuable_meta(@merge_request, @project, "Merge request")
+ = issuable_meta(@merge_request, @project)
%a.btn.btn-default.float-right.d-block.d-sm-none.gutter-toggle.issuable-gutter-toggle.js-sidebar-toggle{ href: "#" }
= sprite_icon('chevron-double-lg-left')
@@ -50,4 +50,4 @@
- if can_update_merge_request && !are_close_and_open_buttons_hidden
= render 'shared/issuable/close_reopen_draft_report_toggle', issuable: @merge_request
- elsif !@merge_request.merged?
- = link_to _('Report abuse'), new_abuse_report_path(user_id: @merge_request.author.id, ref_url: issuable_url(@merge_request)), class: 'gl-display-none gl-display-md-block gl-button btn btn-warning-secondary', title: _('Report abuse')
+ = link_to _('Report abuse'), new_abuse_report_path(user_id: @merge_request.author.id, ref_url: merge_request_url(@merge_request)), class: 'gl-display-none gl-display-md-block gl-button btn btn-warning-secondary', title: _('Report abuse')
diff --git a/app/views/shared/issuable/_close_reopen_draft_report_toggle.html.haml b/app/views/shared/issuable/_close_reopen_draft_report_toggle.html.haml
index bdb53dfe323..250e1516318 100644
--- a/app/views/shared/issuable/_close_reopen_draft_report_toggle.html.haml
+++ b/app/views/shared/issuable/_close_reopen_draft_report_toggle.html.haml
@@ -30,7 +30,7 @@
%li.divider.droplab-item-ignore
%li.report-item
- %a.report-abuse-link{ href: new_abuse_report_path(user_id: issuable.author.id, ref_url: issuable_url(issuable)) }
+ %a.report-abuse-link{ href: new_abuse_report_path(user_id: issuable.author.id, ref_url: merge_request_url(issuable)) }
.description
%strong.title= _('Report abuse')
%p.text
diff --git a/app/views/shared/issue_type/_details_header.html.haml b/app/views/shared/issue_type/_details_header.html.haml
index 7b02d91a314..d6226760ba5 100644
--- a/app/views/shared/issue_type/_details_header.html.haml
+++ b/app/views/shared/issue_type/_details_header.html.haml
@@ -11,7 +11,7 @@
.issuable-meta
#js-issuable-header-warnings
- = issuable_meta(issuable, @project, issuable_display_type(issuable))
+ = issuable_meta(issuable, @project)
%a.btn.gl-button.btn-default.float-right.gl-display-block.d-sm-none.gutter-toggle.issuable-gutter-toggle.js-sidebar-toggle{ href: "#" }
= sprite_icon('chevron-double-lg-left')
diff --git a/changelogs/unreleased/231189-manifest-dir-buttons.yml b/changelogs/unreleased/231189-manifest-dir-buttons.yml
new file mode 100644
index 00000000000..96b581152b3
--- /dev/null
+++ b/changelogs/unreleased/231189-manifest-dir-buttons.yml
@@ -0,0 +1,5 @@
+---
+title: Adds gl button classes to manifest imports
+merge_request: 48697
+author:
+type: other
diff --git a/changelogs/unreleased/267993-enable-by-default-true-in-api-helper-usage_data_-event-feature.yml b/changelogs/unreleased/267993-enable-by-default-true-in-api-helper-usage_data_-event-feature.yml
new file mode 100644
index 00000000000..f3206b40c0a
--- /dev/null
+++ b/changelogs/unreleased/267993-enable-by-default-true-in-api-helper-usage_data_-event-feature.yml
@@ -0,0 +1,5 @@
+---
+title: Enable by default usage data API tracking
+merge_request: 48607
+author:
+type: added
diff --git a/changelogs/unreleased/273777-fix-css-not-loading-on-jira-connect-app.yml b/changelogs/unreleased/273777-fix-css-not-loading-on-jira-connect-app.yml
new file mode 100644
index 00000000000..c9b03dd2f58
--- /dev/null
+++ b/changelogs/unreleased/273777-fix-css-not-loading-on-jira-connect-app.yml
@@ -0,0 +1,5 @@
+---
+title: Fix Jira Connect styles not loaded when startup_css is enabled
+merge_request: 47043
+author:
+type: fixed
diff --git a/changelogs/unreleased/276897-add-external-issue-links.yml b/changelogs/unreleased/276897-add-external-issue-links.yml
new file mode 100644
index 00000000000..8f75c4edd11
--- /dev/null
+++ b/changelogs/unreleased/276897-add-external-issue-links.yml
@@ -0,0 +1,5 @@
+---
+title: Add Vulnerabilities External Link model
+merge_request: 48465
+author:
+type: added
diff --git a/changelogs/unreleased/georgekoltsov-bulk-import-failures.yml b/changelogs/unreleased/georgekoltsov-bulk-import-failures.yml
new file mode 100644
index 00000000000..6d219b39372
--- /dev/null
+++ b/changelogs/unreleased/georgekoltsov-bulk-import-failures.yml
@@ -0,0 +1,6 @@
+---
+title: Add BulkImports::Failure to store import failures of the Group Migration (BulkImports)
+ process
+merge_request: 47526
+author:
+type: changed
diff --git a/config/feature_flags/development/usage_data_a_compliance_audit_events_api.yml b/config/feature_flags/development/usage_data_a_compliance_audit_events_api.yml
index 1daf82b4b8a..9d668c73052 100644
--- a/config/feature_flags/development/usage_data_a_compliance_audit_events_api.yml
+++ b/config/feature_flags/development/usage_data_a_compliance_audit_events_api.yml
@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/233786
milestone: '13.4'
type: development
group: group::compliance
-default_enabled: false
+default_enabled: true
diff --git a/config/feature_flags/development/usage_data_g_compliance_dashboard.yml b/config/feature_flags/development/usage_data_g_compliance_dashboard.yml
index fcef95c2b01..2ca2893b1a9 100644
--- a/config/feature_flags/development/usage_data_g_compliance_dashboard.yml
+++ b/config/feature_flags/development/usage_data_g_compliance_dashboard.yml
@@ -1,8 +1,8 @@
---
name: usage_data_g_compliance_dashboard
-introduced_by_url:
-rollout_issue_url:
-milestone:
+introduced_by_url:
+rollout_issue_url:
+milestone:
type: development
group: group::compliance
-default_enabled: false
+default_enabled: true
diff --git a/db/migrate/20201112132808_create_bulk_import_failures.rb b/db/migrate/20201112132808_create_bulk_import_failures.rb
new file mode 100644
index 00000000000..cdc5a4d6ff0
--- /dev/null
+++ b/db/migrate/20201112132808_create_bulk_import_failures.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class CreateBulkImportFailures < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ with_lock_retries do
+ unless table_exists?(:bulk_import_failures)
+ create_table :bulk_import_failures do |t|
+ t.references :bulk_import_entity,
+ null: false,
+ index: true,
+ foreign_key: { on_delete: :cascade }
+
+ t.datetime_with_timezone :created_at, null: false
+ t.text :pipeline_class, null: false
+ t.text :exception_class, null: false
+ t.text :exception_message, null: false
+ t.text :correlation_id_value, index: true
+ end
+ end
+ end
+
+ add_text_limit :bulk_import_failures, :pipeline_class, 255
+ add_text_limit :bulk_import_failures, :exception_class, 255
+ add_text_limit :bulk_import_failures, :exception_message, 255
+ add_text_limit :bulk_import_failures, :correlation_id_value, 255
+ end
+
+ def down
+ with_lock_retries do
+ drop_table :bulk_import_failures
+ end
+ end
+end
diff --git a/db/migrate/20201124075951_create_vulnerability_external_links.rb b/db/migrate/20201124075951_create_vulnerability_external_links.rb
new file mode 100644
index 00000000000..8200b15559b
--- /dev/null
+++ b/db/migrate/20201124075951_create_vulnerability_external_links.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class CreateVulnerabilityExternalLinks < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ with_lock_retries do
+ create_table :vulnerability_external_issue_links, if_not_exists: true do |t|
+ t.timestamps_with_timezone null: false
+ t.references :author, null: false, index: true, foreign_key: { to_table: :users, on_delete: :nullify }, type: :bigint
+ t.references :vulnerability, null: false, index: true, type: :bigint
+ t.integer :link_type, limit: 2, null: false, default: 1 # 'created'
+ t.integer :external_type, limit: 2, null: false, default: 1 # 'jira'
+ t.text :external_project_key, null: false
+ t.text :external_issue_key, null: false
+
+ t.index %i[vulnerability_id external_type external_project_key external_issue_key],
+ name: 'idx_vulnerability_ext_issue_links_on_vulne_id_and_ext_issue',
+ unique: true
+ t.index %i[vulnerability_id link_type],
+ name: 'idx_vulnerability_ext_issue_links_on_vulne_id_and_link_type',
+ where: 'link_type = 1',
+ unique: true # only one 'created' link per vulnerability is allowed
+ end
+ end
+
+ add_concurrent_foreign_key :vulnerability_external_issue_links, :vulnerabilities, column: :vulnerability_id, on_delete: :cascade
+
+ add_text_limit :vulnerability_external_issue_links, :external_project_key, 255
+ add_text_limit :vulnerability_external_issue_links, :external_issue_key, 255
+ end
+
+ def down
+ with_lock_retries do
+ drop_table :vulnerability_external_issue_links
+ end
+ end
+end
diff --git a/db/schema_migrations/20201112132808 b/db/schema_migrations/20201112132808
new file mode 100644
index 00000000000..d6cc9595c80
--- /dev/null
+++ b/db/schema_migrations/20201112132808
@@ -0,0 +1 @@
+2b30b1ba41a49ce4a81711e6fef1dbcdaf8b76f824aaf83702cd27833815e57b \ No newline at end of file
diff --git a/db/schema_migrations/20201124075951 b/db/schema_migrations/20201124075951
new file mode 100644
index 00000000000..b659c83ad21
--- /dev/null
+++ b/db/schema_migrations/20201124075951
@@ -0,0 +1 @@
+6779e92fa65ff206b19bb99a5a242e3ab5fd7a8d15be89dee925d1fbb5b00632 \ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 053cab899bb..beb05a5c5e6 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -9927,6 +9927,29 @@ CREATE SEQUENCE bulk_import_entities_id_seq
ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id;
+CREATE TABLE bulk_import_failures (
+ id bigint NOT NULL,
+ bulk_import_entity_id bigint NOT NULL,
+ created_at timestamp with time zone NOT NULL,
+ pipeline_class text NOT NULL,
+ exception_class text NOT NULL,
+ exception_message text NOT NULL,
+ correlation_id_value text,
+ CONSTRAINT check_053d65c7a4 CHECK ((char_length(pipeline_class) <= 255)),
+ CONSTRAINT check_6eca8f972e CHECK ((char_length(exception_message) <= 255)),
+ CONSTRAINT check_c7dba8398e CHECK ((char_length(exception_class) <= 255)),
+ CONSTRAINT check_e787285882 CHECK ((char_length(correlation_id_value) <= 255))
+);
+
+CREATE SEQUENCE bulk_import_failures_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE bulk_import_failures_id_seq OWNED BY bulk_import_failures.id;
+
CREATE TABLE bulk_import_trackers (
id bigint NOT NULL,
bulk_import_entity_id bigint NOT NULL,
@@ -17231,6 +17254,29 @@ CREATE SEQUENCE vulnerability_exports_id_seq
ALTER SEQUENCE vulnerability_exports_id_seq OWNED BY vulnerability_exports.id;
+CREATE TABLE vulnerability_external_issue_links (
+ id bigint NOT NULL,
+ created_at timestamp with time zone NOT NULL,
+ updated_at timestamp with time zone NOT NULL,
+ author_id bigint NOT NULL,
+ vulnerability_id bigint NOT NULL,
+ link_type smallint DEFAULT 1 NOT NULL,
+ external_type smallint DEFAULT 1 NOT NULL,
+ external_project_key text NOT NULL,
+ external_issue_key text NOT NULL,
+ CONSTRAINT check_3200604f5e CHECK ((char_length(external_issue_key) <= 255)),
+ CONSTRAINT check_68cffd19b0 CHECK ((char_length(external_project_key) <= 255))
+);
+
+CREATE SEQUENCE vulnerability_external_issue_links_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE vulnerability_external_issue_links_id_seq OWNED BY vulnerability_external_issue_links.id;
+
CREATE TABLE vulnerability_feedback (
id integer NOT NULL,
created_at timestamp with time zone NOT NULL,
@@ -17789,6 +17835,8 @@ ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval(
ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass);
+ALTER TABLE ONLY bulk_import_failures ALTER COLUMN id SET DEFAULT nextval('bulk_import_failures_id_seq'::regclass);
+
ALTER TABLE ONLY bulk_import_trackers ALTER COLUMN id SET DEFAULT nextval('bulk_import_trackers_id_seq'::regclass);
ALTER TABLE ONLY bulk_imports ALTER COLUMN id SET DEFAULT nextval('bulk_imports_id_seq'::regclass);
@@ -18429,6 +18477,8 @@ ALTER TABLE ONLY vulnerabilities ALTER COLUMN id SET DEFAULT nextval('vulnerabil
ALTER TABLE ONLY vulnerability_exports ALTER COLUMN id SET DEFAULT nextval('vulnerability_exports_id_seq'::regclass);
+ALTER TABLE ONLY vulnerability_external_issue_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_external_issue_links_id_seq'::regclass);
+
ALTER TABLE ONLY vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('vulnerability_feedback_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_finding_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_links_id_seq'::regclass);
@@ -18815,6 +18865,9 @@ ALTER TABLE ONLY bulk_import_configurations
ALTER TABLE ONLY bulk_import_entities
ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY bulk_import_failures
+ ADD CONSTRAINT bulk_import_failures_pkey PRIMARY KEY (id);
+
ALTER TABLE ONLY bulk_import_trackers
ADD CONSTRAINT bulk_import_trackers_pkey PRIMARY KEY (id);
@@ -19898,6 +19951,9 @@ ALTER TABLE ONLY vulnerabilities
ALTER TABLE ONLY vulnerability_exports
ADD CONSTRAINT vulnerability_exports_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY vulnerability_external_issue_links
+ ADD CONSTRAINT vulnerability_external_issue_links_pkey PRIMARY KEY (id);
+
ALTER TABLE ONLY vulnerability_feedback
ADD CONSTRAINT vulnerability_feedback_pkey PRIMARY KEY (id);
@@ -20230,6 +20286,10 @@ CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan
CREATE UNIQUE INDEX idx_serverless_domain_cluster_on_clusters_applications_knative ON serverless_domain_cluster USING btree (clusters_applications_knative_id);
+CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_ext_issue ON vulnerability_external_issue_links USING btree (vulnerability_id, external_type, external_project_key, external_issue_key);
+
+CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_link_type ON vulnerability_external_issue_links USING btree (vulnerability_id, link_type) WHERE (link_type = 1);
+
CREATE UNIQUE INDEX idx_vulnerability_issue_links_on_vulnerability_id_and_issue_id ON vulnerability_issue_links USING btree (vulnerability_id, issue_id);
CREATE UNIQUE INDEX idx_vulnerability_issue_links_on_vulnerability_id_and_link_type ON vulnerability_issue_links USING btree (vulnerability_id, link_type) WHERE (link_type = 2);
@@ -20432,6 +20492,10 @@ CREATE INDEX index_bulk_import_entities_on_parent_id ON bulk_import_entities USI
CREATE INDEX index_bulk_import_entities_on_project_id ON bulk_import_entities USING btree (project_id);
+CREATE INDEX index_bulk_import_failures_on_bulk_import_entity_id ON bulk_import_failures USING btree (bulk_import_entity_id);
+
+CREATE INDEX index_bulk_import_failures_on_correlation_id_value ON bulk_import_failures USING btree (correlation_id_value);
+
CREATE INDEX index_bulk_imports_on_user_id ON bulk_imports USING btree (user_id);
CREATE UNIQUE INDEX index_chat_names_on_service_id_and_team_id_and_chat_id ON chat_names USING btree (service_id, team_id, chat_id);
@@ -22396,6 +22460,10 @@ CREATE INDEX index_vulnerability_exports_on_group_id_not_null ON vulnerability_e
CREATE INDEX index_vulnerability_exports_on_project_id_not_null ON vulnerability_exports USING btree (project_id) WHERE (project_id IS NOT NULL);
+CREATE INDEX index_vulnerability_external_issue_links_on_author_id ON vulnerability_external_issue_links USING btree (author_id);
+
+CREATE INDEX index_vulnerability_external_issue_links_on_vulnerability_id ON vulnerability_external_issue_links USING btree (vulnerability_id);
+
CREATE INDEX index_vulnerability_feedback_on_author_id ON vulnerability_feedback USING btree (author_id);
CREATE INDEX index_vulnerability_feedback_on_comment_author_id ON vulnerability_feedback USING btree (comment_author_id);
@@ -23426,6 +23494,9 @@ ALTER TABLE ONLY emails
ALTER TABLE ONLY clusters
ADD CONSTRAINT fk_f05c5e5a42 FOREIGN KEY (management_project_id) REFERENCES projects(id) ON DELETE SET NULL;
+ALTER TABLE ONLY vulnerability_external_issue_links
+ ADD CONSTRAINT fk_f07bb8233d FOREIGN KEY (vulnerability_id) REFERENCES vulnerabilities(id) ON DELETE CASCADE;
+
ALTER TABLE ONLY epics
ADD CONSTRAINT fk_f081aa4489 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
@@ -23606,6 +23677,9 @@ ALTER TABLE ONLY cluster_providers_aws
ALTER TABLE ONLY grafana_integrations
ADD CONSTRAINT fk_rails_18d0e2b564 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+ALTER TABLE ONLY bulk_import_failures
+ ADD CONSTRAINT fk_rails_1964240b8c FOREIGN KEY (bulk_import_entity_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
+
ALTER TABLE ONLY group_wiki_repositories
ADD CONSTRAINT fk_rails_19755e374b FOREIGN KEY (shard_id) REFERENCES shards(id) ON DELETE RESTRICT;
@@ -24644,6 +24718,9 @@ ALTER TABLE ONLY vulnerability_occurrence_identifiers
ALTER TABLE ONLY serverless_domain_cluster
ADD CONSTRAINT fk_rails_e59e868733 FOREIGN KEY (clusters_applications_knative_id) REFERENCES clusters_applications_knative(id) ON DELETE CASCADE;
+ALTER TABLE ONLY vulnerability_external_issue_links
+ ADD CONSTRAINT fk_rails_e5ba7f7b13 FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL;
+
ALTER TABLE ONLY approval_merge_request_rule_sources
ADD CONSTRAINT fk_rails_e605a04f76 FOREIGN KEY (approval_merge_request_rule_id) REFERENCES approval_merge_request_rules(id) ON DELETE CASCADE;
diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md
index f202a912696..5a23157c8fb 100644
--- a/doc/administration/instance_limits.md
+++ b/doc/administration/instance_limits.md
@@ -165,7 +165,7 @@ There is a limit when embedding metrics in GFM for performance reasons.
On GitLab.com, the [maximum number of webhooks and their size](../user/gitlab_com/index.md#webhooks) per project, and per group, is limited.
-To set this limit on a self-managed installation, run the following in the
+To set this limit on a self-managed installation, where the default is `100` project webhooks and `50` group webhooks, run the following in the
[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
```ruby
@@ -173,7 +173,7 @@ To set this limit on a self-managed installation, run the following in the
# Plan.default.create_limits!
# For project webhooks
-Plan.default.actual_limits.update!(project_hooks: 100)
+Plan.default.actual_limits.update!(project_hooks: 200)
# For group webhooks
Plan.default.actual_limits.update!(group_hooks: 100)
@@ -235,8 +235,8 @@ If a new pipeline would cause the total number of jobs to exceed the limit, the
will fail with a `job_activity_limit_exceeded` error.
- On GitLab.com different [limits are defined per plan](../user/gitlab_com/index.md#gitlab-cicd) and they affect all projects under that plan.
-- On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-managed installations, this limit is defined for the `default` plan that affects all projects.
- This limit is disabled by default.
+- On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-managed installations, this limit is defined under a `default` plan that affects all projects.
+ This limit is disabled (`0`) by default.
To set this limit on a self-managed installation, run the following in the
[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
@@ -261,7 +261,7 @@ too many deployments fail with a `deployments_limit_exceeded` error.
The default limit is 500 for all [self-managed and GitLab.com plans](https://about.gitlab.com/pricing/).
-To change the limit on a self-managed installation, change the `default` plan limit with the following
+To change the limit on a self-managed installation, change the `default` plan's limit with the following
[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session) command:
```ruby
@@ -284,7 +284,7 @@ If a new subscription would cause the total number of subscription to exceed the
limit, the subscription will be considered invalid.
- On GitLab.com different [limits are defined per plan](../user/gitlab_com/index.md#gitlab-cicd) and they affect all projects under that plan.
-- On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-managed installations, this limit is defined for the `default` plan that affects all projects.
+- On [GitLab Starter](https://about.gitlab.com/pricing/#self-managed) tier or higher self-managed installations, this limit is defined under a `default` plan that affects all projects. By default, there is a limit of `2` subscriptions.
To set this limit on a self-managed installation, run the following in the
[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
@@ -308,8 +308,8 @@ On GitLab.com, different limits are [defined per plan](../user/gitlab_com/index.
and they affect all projects under that plan.
On self-managed instances ([GitLab Starter](https://about.gitlab.com/pricing/#self-managed)
-or higher tiers), this limit is defined for the `default` plan that affects all
-projects. By default, there is no limit.
+or higher tiers), this limit is defined under a `default` plan that affects all
+projects. By default, there is a limit of `10` pipeline schedules.
To set this limit on a self-managed installation, run the following in the
[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session):
diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md
index f446ff9f693..6c6533e1d11 100644
--- a/doc/administration/reference_architectures/10k_users.md
+++ b/doc/administration/reference_architectures/10k_users.md
@@ -33,6 +33,81 @@ full list of reference architectures, see
| Object storage | n/a | n/a | n/a | n/a | n/a |
| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 |
+```mermaid
+stateDiagram-v2
+ [*] --> LoadBalancer
+ LoadBalancer --> ApplicationServer
+
+ ApplicationServer --> BackgroundJobs
+ ApplicationServer --> Gitaly
+ ApplicationServer --> Redis_Cache
+ ApplicationServer --> Redis_Queues
+ ApplicationServer --> PgBouncer
+ PgBouncer --> Database
+ ApplicationServer --> ObjectStorage
+ BackgroundJobs --> ObjectStorage
+
+ ApplicationMonitoring -->ApplicationServer
+ ApplicationMonitoring -->PgBouncer
+ ApplicationMonitoring -->Database
+ ApplicationMonitoring -->BackgroundJobs
+
+ ApplicationServer --> Consul
+
+ Consul --> Database
+ Consul --> PgBouncer
+ Redis_Cache --> Consul
+ Redis_Queues --> Consul
+ BackgroundJobs --> Consul
+
+ state Consul {
+ "Consul_1..3"
+ }
+
+ state Database {
+ "PG_Primary_Node"
+ "PG_Secondary_Node_1..2"
+ }
+
+ state Redis_Cache {
+ "R_Cache_Primary_Node"
+ "R_Cache_Replica_Node_1..2"
+ "R_Cache_Sentinel_1..3"
+ }
+
+ state Redis_Queues {
+ "R_Queues_Primary_Node"
+ "R_Queues_Replica_Node_1..2"
+ "R_Queues_Sentinel_1..3"
+ }
+
+ state Gitaly {
+ "Gitaly_1..2"
+ }
+
+ state BackgroundJobs {
+ "Sidekiq_1..4"
+ }
+
+ state ApplicationServer {
+ "GitLab_Rails_1..3"
+ }
+
+ state LoadBalancer {
+ "LoadBalancer_1"
+ }
+
+ state ApplicationMonitoring {
+ "Prometheus"
+ "Grafana"
+ }
+
+ state PgBouncer {
+ "Internal_Load_Balancer"
+ "PgBouncer_1..3"
+ }
+```
+
The Google Cloud Platform (GCP) architectures were built and tested using the
[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform. On different hardware you may find that adjustments, either lower
diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md
index 0a9a880de18..d0cced31ae7 100644
--- a/doc/administration/reference_architectures/25k_users.md
+++ b/doc/administration/reference_architectures/25k_users.md
@@ -33,6 +33,81 @@ full list of reference architectures, see
| Object storage | n/a | n/a | n/a | n/a | n/a |
| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 |
+```mermaid
+stateDiagram-v2
+ [*] --> LoadBalancer
+ LoadBalancer --> ApplicationServer
+
+ ApplicationServer --> BackgroundJobs
+ ApplicationServer --> Gitaly
+ ApplicationServer --> Redis_Cache
+ ApplicationServer --> Redis_Queues
+ ApplicationServer --> PgBouncer
+ PgBouncer --> Database
+ ApplicationServer --> ObjectStorage
+ BackgroundJobs --> ObjectStorage
+
+ ApplicationMonitoring -->ApplicationServer
+ ApplicationMonitoring -->PgBouncer
+ ApplicationMonitoring -->Database
+ ApplicationMonitoring -->BackgroundJobs
+
+ ApplicationServer --> Consul
+
+ Consul --> Database
+ Consul --> PgBouncer
+ Redis_Cache --> Consul
+ Redis_Queues --> Consul
+ BackgroundJobs --> Consul
+
+ state Consul {
+ "Consul_1..3"
+ }
+
+ state Database {
+ "PG_Primary_Node"
+ "PG_Secondary_Node_1..2"
+ }
+
+ state Redis_Cache {
+ "R_Cache_Primary_Node"
+ "R_Cache_Replica_Node_1..2"
+ "R_Cache_Sentinel_1..3"
+ }
+
+ state Redis_Queues {
+ "R_Queues_Primary_Node"
+ "R_Queues_Replica_Node_1..2"
+ "R_Queues_Sentinel_1..3"
+ }
+
+ state Gitaly {
+ "Gitaly_1..2"
+ }
+
+ state BackgroundJobs {
+ "Sidekiq_1..4"
+ }
+
+ state ApplicationServer {
+ "GitLab_Rails_1..5"
+ }
+
+ state LoadBalancer {
+ "LoadBalancer_1"
+ }
+
+ state ApplicationMonitoring {
+ "Prometheus"
+ "Grafana"
+ }
+
+ state PgBouncer {
+ "Internal_Load_Balancer"
+ "PgBouncer_1..3"
+ }
+```
+
The Google Cloud Platform (GCP) architectures were built and tested using the
[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform. On different hardware you may find that adjustments, either lower
diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md
index e72242c9f90..ea3483f2ad2 100644
--- a/doc/administration/reference_architectures/2k_users.md
+++ b/doc/administration/reference_architectures/2k_users.md
@@ -26,6 +26,46 @@ For a full list of reference architectures, see
| Object storage | n/a | n/a | n/a | n/a | n/a |
| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 |
+```mermaid
+stateDiagram-v2
+ [*] --> LoadBalancer
+ LoadBalancer --> ApplicationServer
+
+ ApplicationServer --> Gitaly
+ ApplicationServer --> Redis
+ ApplicationServer --> Database
+ ApplicationServer --> ObjectStorage
+
+ ApplicationMonitoring -->ApplicationServer
+ ApplicationMonitoring -->Redis
+ ApplicationMonitoring -->Database
+
+
+ state Database {
+ "PG_Node"
+ }
+ state Redis {
+ "Redis_Node"
+ }
+
+ state Gitaly {
+ "Gitaly"
+ }
+
+ state ApplicationServer {
+ "AppServ_1..2"
+ }
+
+ state LoadBalancer {
+ "LoadBalancer"
+ }
+
+ state ApplicationMonitoring {
+ "Prometheus"
+ "Grafana"
+ }
+```
+
The Google Cloud Platform (GCP) architectures were built and tested using the
[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform. On different hardware you may find that adjustments, either lower
diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md
index b2c000cb3aa..529f0252862 100644
--- a/doc/administration/reference_architectures/3k_users.md
+++ b/doc/administration/reference_architectures/3k_users.md
@@ -37,6 +37,62 @@ costly-to-operate environment by using the
| Object storage | n/a | n/a | n/a | n/a | n/a |
| NFS server (optional, not recommended) | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 |
+```mermaid
+stateDiagram-v2
+ [*] --> LoadBalancer
+ LoadBalancer --> ApplicationServer
+
+ ApplicationServer --> BackgroundJobs
+ ApplicationServer --> Gitaly
+ ApplicationServer --> Redis
+ ApplicationServer --> PgBouncer
+ PgBouncer --> Database
+ ApplicationServer --> ObjectStorage
+ BackgroundJobs --> ObjectStorage
+
+ ApplicationMonitoring -->ApplicationServer
+ ApplicationMonitoring -->Redis
+ ApplicationMonitoring -->PgBouncer
+ ApplicationMonitoring -->Database
+ ApplicationMonitoring -->BackgroundJobs
+
+ state Database {
+ "PG_Primary_Node"
+ "PG_Secondary_Node_1..2"
+ }
+ state Redis {
+ "R_Primary_Node"
+ "R_Replica_Node_1..2"
+ "R_Consul/Sentinel_1..3"
+ }
+
+ state Gitaly {
+ "Gitaly_1..2"
+ }
+
+ state BackgroundJobs {
+ "Sidekiq_1..4"
+ }
+
+ state ApplicationServer {
+ "GitLab_Rails_1..3"
+ }
+
+ state LoadBalancer {
+ "LoadBalancer_1"
+ }
+
+ state ApplicationMonitoring {
+ "Prometheus"
+ "Grafana"
+ }
+
+ state PgBouncer {
+ "Internal_Load_Balancer"
+ "PgBouncer_1..3"
+ }
+```
+
The Google Cloud Platform (GCP) architectures were built and tested using the
[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform. On different hardware you may find that adjustments, either lower
diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md
index 7eb842ff6e5..3ef75c16c92 100644
--- a/doc/administration/reference_architectures/50k_users.md
+++ b/doc/administration/reference_architectures/50k_users.md
@@ -33,6 +33,81 @@ full list of reference architectures, see
| Object storage | n/a | n/a | n/a | n/a | n/a |
| NFS server | 1 | 4 vCPU, 3.6 GB memory | n1-highcpu-4 | c5.xlarge | F4s v2 |
+```mermaid
+stateDiagram-v2
+ [*] --> LoadBalancer
+ LoadBalancer --> ApplicationServer
+
+ ApplicationServer --> BackgroundJobs
+ ApplicationServer --> Gitaly
+ ApplicationServer --> Redis_Cache
+ ApplicationServer --> Redis_Queues
+ ApplicationServer --> PgBouncer
+ PgBouncer --> Database
+ ApplicationServer --> ObjectStorage
+ BackgroundJobs --> ObjectStorage
+
+ ApplicationMonitoring -->ApplicationServer
+ ApplicationMonitoring -->PgBouncer
+ ApplicationMonitoring -->Database
+ ApplicationMonitoring -->BackgroundJobs
+
+ ApplicationServer --> Consul
+
+ Consul --> Database
+ Consul --> PgBouncer
+ Redis_Cache --> Consul
+ Redis_Queues --> Consul
+ BackgroundJobs --> Consul
+
+ state Consul {
+ "Consul_1..3"
+ }
+
+ state Database {
+ "PG_Primary_Node"
+ "PG_Secondary_Node_1..2"
+ }
+
+ state Redis_Cache {
+ "R_Cache_Primary_Node"
+ "R_Cache_Replica_Node_1..2"
+ "R_Cache_Sentinel_1..3"
+ }
+
+ state Redis_Queues {
+ "R_Queues_Primary_Node"
+ "R_Queues_Replica_Node_1..2"
+ "R_Queues_Sentinel_1..3"
+ }
+
+ state Gitaly {
+ "Gitaly_1..2"
+ }
+
+ state BackgroundJobs {
+ "Sidekiq_1..4"
+ }
+
+ state ApplicationServer {
+ "GitLab_Rails_1..12"
+ }
+
+ state LoadBalancer {
+ "LoadBalancer_1"
+ }
+
+ state ApplicationMonitoring {
+ "Prometheus"
+ "Grafana"
+ }
+
+ state PgBouncer {
+ "Internal_Load_Balancer"
+ "PgBouncer_1..3"
+ }
+```
+
The Google Cloud Platform (GCP) architectures were built and tested using the
[Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform. On different hardware you may find that adjustments, either lower
diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md
index 877c77ee2b1..3dfe614a495 100644
--- a/doc/administration/reference_architectures/5k_users.md
+++ b/doc/administration/reference_architectures/5k_users.md
@@ -60,6 +60,7 @@ stateDiagram-v2
"PG_Primary_Node"
"PG_Secondary_Node_1..2"
}
+
state Redis {
"R_Primary_Node"
"R_Replica_Node_1..2"
@@ -90,7 +91,6 @@ stateDiagram-v2
state PgBouncer {
"Internal_Load_Balancer"
"PgBouncer_1..3"
-
}
```
diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql
index 4423c513359..0819bb2973b 100644
--- a/doc/api/graphql/reference/gitlab_schema.graphql
+++ b/doc/api/graphql/reference/gitlab_schema.graphql
@@ -14057,6 +14057,7 @@ type Mutation {
mergeRequestUpdate(input: MergeRequestUpdateInput!): MergeRequestUpdatePayload
namespaceIncreaseStorageTemporarily(input: NamespaceIncreaseStorageTemporarilyInput!): NamespaceIncreaseStorageTemporarilyPayload
oncallScheduleCreate(input: OncallScheduleCreateInput!): OncallScheduleCreatePayload
+ oncallScheduleDestroy(input: OncallScheduleDestroyInput!): OncallScheduleDestroyPayload
pipelineCancel(input: PipelineCancelInput!): PipelineCancelPayload
pipelineDestroy(input: PipelineDestroyInput!): PipelineDestroyPayload
pipelineRetry(input: PipelineRetryInput!): PipelineRetryPayload
@@ -14685,6 +14686,46 @@ type OncallScheduleCreatePayload {
}
"""
+Autogenerated input type of OncallScheduleDestroy
+"""
+input OncallScheduleDestroyInput {
+ """
+ A unique identifier for the client performing the mutation.
+ """
+ clientMutationId: String
+
+ """
+ The on-call schedule internal ID to remove
+ """
+ iid: String!
+
+ """
+ The project to remove the on-call schedule from
+ """
+ projectPath: ID!
+}
+
+"""
+Autogenerated return type of OncallScheduleDestroy
+"""
+type OncallScheduleDestroyPayload {
+ """
+ A unique identifier for the client performing the mutation.
+ """
+ clientMutationId: String
+
+ """
+ Errors encountered during execution of the mutation.
+ """
+ errors: [String!]!
+
+ """
+ The on-call schedule
+ """
+ oncallSchedule: IncidentManagementOncallSchedule
+}
+
+"""
Represents a package
"""
type Package {
diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json
index d552da0b537..8ed969e512e 100644
--- a/doc/api/graphql/reference/gitlab_schema.json
+++ b/doc/api/graphql/reference/gitlab_schema.json
@@ -40868,6 +40868,33 @@
"deprecationReason": null
},
{
+ "name": "oncallScheduleDestroy",
+ "description": null,
+ "args": [
+ {
+ "name": "input",
+ "description": null,
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "INPUT_OBJECT",
+ "name": "OncallScheduleDestroyInput",
+ "ofType": null
+ }
+ },
+ "defaultValue": null
+ }
+ ],
+ "type": {
+ "kind": "OBJECT",
+ "name": "OncallScheduleDestroyPayload",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
"name": "pipelineCancel",
"description": null,
"args": [
@@ -43618,6 +43645,122 @@
"possibleTypes": null
},
{
+ "kind": "INPUT_OBJECT",
+ "name": "OncallScheduleDestroyInput",
+ "description": "Autogenerated input type of OncallScheduleDestroy",
+ "fields": null,
+ "inputFields": [
+ {
+ "name": "projectPath",
+ "description": "The project to remove the on-call schedule from",
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "SCALAR",
+ "name": "ID",
+ "ofType": null
+ }
+ },
+ "defaultValue": null
+ },
+ {
+ "name": "iid",
+ "description": "The on-call schedule internal ID to remove",
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ }
+ },
+ "defaultValue": null
+ },
+ {
+ "name": "clientMutationId",
+ "description": "A unique identifier for the client performing the mutation.",
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "defaultValue": null
+ }
+ ],
+ "interfaces": null,
+ "enumValues": null,
+ "possibleTypes": null
+ },
+ {
+ "kind": "OBJECT",
+ "name": "OncallScheduleDestroyPayload",
+ "description": "Autogenerated return type of OncallScheduleDestroy",
+ "fields": [
+ {
+ "name": "clientMutationId",
+ "description": "A unique identifier for the client performing the mutation.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "errors",
+ "description": "Errors encountered during execution of the mutation.",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "LIST",
+ "name": null,
+ "ofType": {
+ "kind": "NON_NULL",
+ "name": null,
+ "ofType": {
+ "kind": "SCALAR",
+ "name": "String",
+ "ofType": null
+ }
+ }
+ }
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ },
+ {
+ "name": "oncallSchedule",
+ "description": "The on-call schedule",
+ "args": [
+
+ ],
+ "type": {
+ "kind": "OBJECT",
+ "name": "IncidentManagementOncallSchedule",
+ "ofType": null
+ },
+ "isDeprecated": false,
+ "deprecationReason": null
+ }
+ ],
+ "inputFields": null,
+ "interfaces": [
+
+ ],
+ "enumValues": null,
+ "possibleTypes": null
+ },
+ {
"kind": "OBJECT",
"name": "Package",
"description": "Represents a package",
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index 21d6d020779..5dfda50285e 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -2237,6 +2237,16 @@ Autogenerated return type of OncallScheduleCreate.
| `errors` | String! => Array | Errors encountered during execution of the mutation. |
| `oncallSchedule` | IncidentManagementOncallSchedule | The on-call schedule |
+### OncallScheduleDestroyPayload
+
+Autogenerated return type of OncallScheduleDestroy.
+
+| Field | Type | Description |
+| ----- | ---- | ----------- |
+| `clientMutationId` | String | A unique identifier for the client performing the mutation. |
+| `errors` | String! => Array | Errors encountered during execution of the mutation. |
+| `oncallSchedule` | IncidentManagementOncallSchedule | The on-call schedule |
+
### Package
Represents a package.
diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md
index e51cb8a08cd..b763cd3f1cf 100644
--- a/doc/development/fe_guide/graphql.md
+++ b/doc/development/fe_guide/graphql.md
@@ -972,14 +972,100 @@ it('calls a mutation with correct parameters and reorders designs', async () =>
#### Testing `@client` queries
-If your application contains `@client` queries, most probably you will have an Apollo Client warning saying that you have a local query but no resolvers are defined. In order to fix it, you need to pass resolvers to the mocked client with a second parameter (bare minimum is an empty object):
+##### Using mock resolvers
+
+If your application contains `@client` queries, you get
+the following Apollo Client warning when passing only handlers:
+
+```shell
+Unexpected call of console.warn() with:
+
+Warning: mock-apollo-client - The query is entirely client-side (using @client directives) and resolvers have been configured. The request handler will not be called.
+```
+
+To fix this you should define mock `resolvers` instead of
+mock `handlers`. For example, given the following `@client` query:
+
+```graphql
+query getBlobContent($path: String, $ref: String!) {
+ blobContent(path: $path, ref: $ref) @client {
+ rawData
+ }
+}
+```
+
+And its actual client-side resolvers:
```javascript
-import createMockApollo from 'jest/helpers/mock_apollo_helper';
-...
-const mockApollo = createMockApollo(requestHandlers, resolvers);
+import Api from '~/api';
+
+export const resolvers = {
+ Query: {
+ blobContent(_, { path, ref }) {
+ return {
+ __typename: 'BlobContent',
+ rawData: Api.getRawFile(path, { ref }).then(({ data }) => {
+ return data;
+ }),
+ };
+ },
+ },
+};
+
+export default resolvers;
```
+We can use a **mock resolver** that returns data with the
+same shape, while mock the result with a mock function:
+
+```javascript
+let mockApollo;
+let mockBlobContentData; // mock function, jest.fn();
+
+const mockResolvers = {
+ Query: {
+ blobContent() {
+ return {
+ __typename: 'BlobContent',
+ rawData: mockBlobContentData(), // the mock function can resolve mock data
+ };
+ },
+ },
+};
+
+const createComponentWithApollo = ({ props = {} } = {}) => {
+ mockApollo = createMockApollo([], mockResolvers); // resolvers are the second parameter
+
+ wrapper = shallowMount(MyComponent, {
+ localVue,
+ propsData: {},
+ apolloProvider: mockApollo,
+ // ...
+ })
+};
+
+```
+
+After which, you can resolve or reject the value needed.
+
+```javascript
+beforeEach(() => {
+ mockBlobContentData = jest.fn();
+});
+
+it('shows data', async() => {
+ mockBlobContentData.mockResolvedValue(data); // you may resolve or reject to mock the result
+
+ createComponentWithApollo();
+
+ await waitForPromises(); // wait on the resolver mock to execute
+
+ expect(findContent().text()).toBe(mockCiYml);
+});
+```
+
+##### Using `cache.writeQuery`
+
Sometimes we want to test a `result` hook of the local query. In order to have it triggered, we need to populate a cache with correct data to be fetched with this query:
```javascript
diff --git a/doc/development/product_analytics/snowplow.md b/doc/development/product_analytics/snowplow.md
index 20786a63b54..b864e48fa5b 100644
--- a/doc/development/product_analytics/snowplow.md
+++ b/doc/development/product_analytics/snowplow.md
@@ -112,7 +112,7 @@ The current method provides several attributes that are sent on each click event
### Web-specific parameters
-Snowplow JS adds many [web-specific parameters](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/snowplow-tracker-protocol/#21_Web-specific_parameters) to all web events by default.
+Snowplow JS adds many [web-specific parameters](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/snowplow-tracker-protocol/#Web-specific_parameters) to all web events by default.
## Implementing Snowplow JS (Frontend) tracking
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 3fa743d70fb..42d7c09c7ea 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -550,8 +550,9 @@ module API
def increment_unique_values(event_name, values)
return unless values.present?
- feature_name = "usage_data_#{event_name}"
- return unless Feature.enabled?(feature_name)
+ feature_flag = "usage_data_#{event_name}"
+
+ return unless Feature.enabled?(feature_flag, default_enabled: true)
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name)
rescue => error
diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb
index 75af126ebf4..840e97ece8a 100644
--- a/lib/bulk_imports/groups/pipelines/group_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb
@@ -6,6 +6,8 @@ module BulkImports
class GroupPipeline
include Pipeline
+ abort_on_failure!
+
extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery
transformer Common::Transformers::HashKeyDigger, key_path: %w[data group]
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 4b4690a43e9..a44f8fc7193 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -26,13 +26,17 @@ module BulkImports
@after_run ||= self.class.after_run_callback
end
- def pipeline_name
+ def pipeline
@pipeline ||= self.class.name
end
def instantiate(class_config)
class_config[:klass].new(class_config[:options])
end
+
+ def abort_on_failure?
+ self.class.abort_on_failure?
+ end
end
class_methods do
@@ -68,6 +72,14 @@ module BulkImports
class_attributes[:after_run]
end
+ def abort_on_failure!
+ class_attributes[:abort_on_failure] = true
+ end
+
+ def abort_on_failure?
+ class_attributes[:abort_on_failure]
+ end
+
private
def add_attribute(sym, klass, options)
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index 898540d42ed..88b96f0ab6e 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -5,35 +5,102 @@ module BulkImports
module Runner
extend ActiveSupport::Concern
+ MarkedAsFailedError = Class.new(StandardError)
+
def run(context)
- info(context, message: "Pipeline started", pipeline: pipeline_name)
+ raise MarkedAsFailedError if marked_as_failed?(context)
+
+ info(context, message: 'Pipeline started', pipeline_class: pipeline)
extractors.each do |extractor|
- extractor.extract(context).each do |entry|
- info(context, extractor: extractor.class.name)
+ data = run_pipeline_step(:extractor, extractor.class.name, context) do
+ extractor.extract(context)
+ end
- transformers.each do |transformer|
- info(context, transformer: transformer.class.name)
- entry = transformer.transform(context, entry)
- end
+ if data && data.respond_to?(:each)
+ data.each do |entry|
+ transformers.each do |transformer|
+ entry = run_pipeline_step(:transformer, transformer.class.name, context) do
+ transformer.transform(context, entry)
+ end
+ end
- loaders.each do |loader|
- info(context, loader: loader.class.name)
- loader.load(context, entry)
+ loaders.each do |loader|
+ run_pipeline_step(:loader, loader.class.name, context) do
+ loader.load(context, entry)
+ end
+ end
end
end
end
after_run.call(context) if after_run.present?
+ rescue MarkedAsFailedError
+ log_skip(context)
end
private # rubocop:disable Lint/UselessAccessModifier
+ def run_pipeline_step(type, class_name, context)
+ raise MarkedAsFailedError if marked_as_failed?(context)
+
+ info(context, type => class_name)
+
+ yield
+ rescue MarkedAsFailedError
+ log_skip(context, type => class_name)
+ rescue => e
+ log_import_failure(e, context)
+
+ mark_as_failed(context) if abort_on_failure?
+ end
+
+ def mark_as_failed(context)
+ warn(context, message: 'Pipeline failed', pipeline_class: pipeline)
+
+ context.entity.fail_op!
+ end
+
+ def marked_as_failed?(context)
+ return true if context.entity.failed?
+
+ false
+ end
+
+ def log_skip(context, extra = {})
+ log = {
+ message: 'Skipping due to failed pipeline status',
+ pipeline_class: pipeline
+ }.merge(extra)
+
+ info(context, log)
+ end
+
+ def log_import_failure(exception, context)
+ attributes = {
+ bulk_import_entity_id: context.entity.id,
+ pipeline_class: pipeline,
+ exception_class: exception.class.to_s,
+ exception_message: exception.message.truncate(255),
+ correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id
+ }
+
+ BulkImports::Failure.create(attributes)
+ end
+
+ def warn(context, extra = {})
+ logger.warn(log_base_params(context).merge(extra))
+ end
+
def info(context, extra = {})
- logger.info({
- entity: context.entity.id,
- entity_type: context.entity.source_type
- }.merge(extra))
+ logger.info(log_base_params(context).merge(extra))
+ end
+
+ def log_base_params(context)
+ {
+ bulk_import_entity_id: context.entity.id,
+ bulk_import_entity_type: context.entity.source_type
+ }
end
def logger
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index fd5c0d99eb3..465c32ca4ea 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -31242,6 +31242,9 @@ msgstr ""
msgid "You have insufficient permissions to create an on-call schedule for this project"
msgstr ""
+msgid "You have insufficient permissions to remove an on-call schedule from this project"
+msgstr ""
+
msgid "You have insufficient permissions to remove this HTTP integration"
msgstr ""
diff --git a/package.json b/package.json
index 4271237ea5b..fe7478d0dee 100644
--- a/package.json
+++ b/package.json
@@ -49,8 +49,8 @@
"@rails/actioncable": "^6.0.3-3",
"@rails/ujs": "^6.0.3-2",
"@sourcegraph/code-host-integration": "0.0.52",
- "@toast-ui/editor": "^2.5.0",
- "@toast-ui/vue-editor": "^2.5.0",
+ "@toast-ui/editor": "^2.5.1",
+ "@toast-ui/vue-editor": "^2.5.1",
"apollo-cache-inmemory": "^1.6.6",
"apollo-client": "^2.6.10",
"apollo-link": "^1.2.14",
diff --git a/spec/factories/bulk_import/failures.rb b/spec/factories/bulk_import/failures.rb
new file mode 100644
index 00000000000..1ebdfdd6c42
--- /dev/null
+++ b/spec/factories/bulk_import/failures.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+FactoryBot.define do
+ factory :bulk_import_failure, class: 'BulkImports::Failure' do
+ association :entity, factory: :bulk_import_entity
+
+ pipeline_class { 'BulkImports::TestPipeline' }
+ exception_class { 'StandardError' }
+ exception_message { 'Standard Error Message' }
+ correlation_id_value { SecureRandom.uuid }
+ end
+end
diff --git a/spec/frontend/fixtures/issues.rb b/spec/frontend/fixtures/issues.rb
index 2c380ba6a96..fccd0ca1d2c 100644
--- a/spec/frontend/fixtures/issues.rb
+++ b/spec/frontend/fixtures/issues.rb
@@ -45,12 +45,6 @@ RSpec.describe Projects::IssuesController, '(JavaScript fixtures)', type: :contr
render_issue(issue)
end
- it 'issues/issue_with_comment.html' do
- issue = create(:issue, project: project)
- create(:note, project: project, noteable: issue, note: '- [ ] Task List Item').save
- render_issue(issue)
- end
-
it 'issues/issue_list.html' do
create(:issue, project: project)
diff --git a/spec/lib/bulk_imports/importers/group_importer_spec.rb b/spec/lib/bulk_imports/importers/group_importer_spec.rb
index 7fec3b79179..95dca7fc486 100644
--- a/spec/lib/bulk_imports/importers/group_importer_spec.rb
+++ b/spec/lib/bulk_imports/importers/group_importer_spec.rb
@@ -23,7 +23,7 @@ RSpec.describe BulkImports::Importers::GroupImporter do
end
describe '#execute' do
- it "starts the entity and run its pipelines" do
+ it 'starts the entity and run its pipelines' do
expect(bulk_import_entity).to receive(:start).and_call_original
expect_to_run_pipeline BulkImports::Groups::Pipelines::GroupPipeline, context: context
expect_to_run_pipeline BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, context: context
@@ -32,6 +32,18 @@ RSpec.describe BulkImports::Importers::GroupImporter do
expect(bulk_import_entity.reload).to be_finished
end
+
+ context 'when failed' do
+ let(:bulk_import_entity) { create(:bulk_import_entity, :failed, bulk_import: bulk_import) }
+
+ it 'does not transition entity to finished state' do
+ allow(bulk_import_entity).to receive(:start!)
+
+ subject.execute
+
+ expect(bulk_import_entity.reload).to be_failed
+ end
+ end
end
def expect_to_run_pipeline(klass, context:)
diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb
index 8c882c799ec..60833e83dcc 100644
--- a/spec/lib/bulk_imports/pipeline/runner_spec.rb
+++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb
@@ -3,26 +3,32 @@
require 'spec_helper'
RSpec.describe BulkImports::Pipeline::Runner do
- describe 'pipeline runner' do
- before do
- extractor = Class.new do
- def initialize(options = {}); end
+ let(:extractor) do
+ Class.new do
+ def initialize(options = {}); end
- def extract(context); end
- end
+ def extract(context); end
+ end
+ end
- transformer = Class.new do
- def initialize(options = {}); end
+ let(:transformer) do
+ Class.new do
+ def initialize(options = {}); end
- def transform(context, entry); end
- end
+ def transform(context); end
+ end
+ end
- loader = Class.new do
- def initialize(options = {}); end
+ let(:loader) do
+ Class.new do
+ def initialize(options = {}); end
- def load(context, entry); end
- end
+ def load(context); end
+ end
+ end
+ describe 'pipeline runner' do
+ before do
stub_const('BulkImports::Extractor', extractor)
stub_const('BulkImports::Transformer', transformer)
stub_const('BulkImports::Loader', loader)
@@ -38,37 +44,126 @@ RSpec.describe BulkImports::Pipeline::Runner do
stub_const('BulkImports::MyPipeline', pipeline)
end
- it 'runs pipeline extractor, transformer, loader' do
- context = instance_double(
- BulkImports::Pipeline::Context,
- entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group')
- )
- entries = [{ foo: :bar }]
-
- expect_next_instance_of(BulkImports::Extractor) do |extractor|
- expect(extractor).to receive(:extract).with(context).and_return(entries)
+ context 'when entity is not marked as failed' do
+ let(:context) do
+ instance_double(
+ BulkImports::Pipeline::Context,
+ entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group', failed?: false)
+ )
end
- expect_next_instance_of(BulkImports::Transformer) do |transformer|
- expect(transformer).to receive(:transform).with(context, entries.first).and_return(entries.first)
+ it 'runs pipeline extractor, transformer, loader' do
+ entries = [{ foo: :bar }]
+
+ expect_next_instance_of(BulkImports::Extractor) do |extractor|
+ expect(extractor).to receive(:extract).with(context).and_return(entries)
+ end
+
+ expect_next_instance_of(BulkImports::Transformer) do |transformer|
+ expect(transformer).to receive(:transform).with(context, entries.first).and_return(entries.first)
+ end
+
+ expect_next_instance_of(BulkImports::Loader) do |loader|
+ expect(loader).to receive(:load).with(context, entries.first)
+ end
+
+ expect_next_instance_of(Gitlab::Import::Logger) do |logger|
+ expect(logger).to receive(:info)
+ .with(
+ message: 'Pipeline started',
+ pipeline_class: 'BulkImports::MyPipeline',
+ bulk_import_entity_id: 1,
+ bulk_import_entity_type: 'group'
+ )
+ expect(logger).to receive(:info)
+ .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', extractor: 'BulkImports::Extractor')
+ expect(logger).to receive(:info)
+ .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', transformer: 'BulkImports::Transformer')
+ expect(logger).to receive(:info)
+ .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', loader: 'BulkImports::Loader')
+ end
+
+ BulkImports::MyPipeline.new.run(context)
end
- expect_next_instance_of(BulkImports::Loader) do |loader|
- expect(loader).to receive(:load).with(context, entries.first)
+ context 'when exception is raised' do
+ let(:entity) { create(:bulk_import_entity, :created) }
+ let(:context) { BulkImports::Pipeline::Context.new(entity: entity) }
+
+ before do
+ allow_next_instance_of(BulkImports::Extractor) do |extractor|
+ allow(extractor).to receive(:extract).with(context).and_raise(StandardError, 'Error!')
+ end
+ end
+
+ it 'logs import failure' do
+ BulkImports::MyPipeline.new.run(context)
+
+ failure = entity.failures.first
+
+ expect(failure).to be_present
+ expect(failure.pipeline_class).to eq('BulkImports::MyPipeline')
+ expect(failure.exception_class).to eq('StandardError')
+ expect(failure.exception_message).to eq('Error!')
+ end
+
+ context 'when pipeline is marked to abort on failure' do
+ before do
+ BulkImports::MyPipeline.abort_on_failure!
+ end
+
+ it 'marks entity as failed' do
+ BulkImports::MyPipeline.new.run(context)
+
+ expect(entity.failed?).to eq(true)
+ end
+
+ it 'logs warn message' do
+ expect_next_instance_of(Gitlab::Import::Logger) do |logger|
+ expect(logger).to receive(:warn)
+ .with(
+ message: 'Pipeline failed',
+ pipeline_class: 'BulkImports::MyPipeline',
+ bulk_import_entity_id: entity.id,
+ bulk_import_entity_type: entity.source_type
+ )
+ end
+
+ BulkImports::MyPipeline.new.run(context)
+ end
+ end
+
+ context 'when pipeline is not marked to abort on failure' do
+ it 'marks entity as failed' do
+ BulkImports::MyPipeline.new.run(context)
+
+ expect(entity.failed?).to eq(false)
+ end
+ end
end
+ end
- expect_next_instance_of(Gitlab::Import::Logger) do |logger|
- expect(logger).to receive(:info)
- .with(message: "Pipeline started", pipeline: 'BulkImports::MyPipeline', entity: 1, entity_type: 'group')
- expect(logger).to receive(:info)
- .with(entity: 1, entity_type: 'group', extractor: 'BulkImports::Extractor')
- expect(logger).to receive(:info)
- .with(entity: 1, entity_type: 'group', transformer: 'BulkImports::Transformer')
- expect(logger).to receive(:info)
- .with(entity: 1, entity_type: 'group', loader: 'BulkImports::Loader')
+ context 'when entity is marked as failed' do
+ let(:context) do
+ instance_double(
+ BulkImports::Pipeline::Context,
+ entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group', failed?: true)
+ )
end
- BulkImports::MyPipeline.new.run(context)
+ it 'logs and returns without execution' do
+ expect_next_instance_of(Gitlab::Import::Logger) do |logger|
+ expect(logger).to receive(:info)
+ .with(
+ message: 'Skipping due to failed pipeline status',
+ pipeline_class: 'BulkImports::MyPipeline',
+ bulk_import_entity_id: 1,
+ bulk_import_entity_type: 'group'
+ )
+ end
+
+ BulkImports::MyPipeline.new.run(context)
+ end
end
end
end
diff --git a/spec/lib/bulk_imports/pipeline_spec.rb b/spec/lib/bulk_imports/pipeline_spec.rb
index ba7bcf8788f..94052be7df2 100644
--- a/spec/lib/bulk_imports/pipeline_spec.rb
+++ b/spec/lib/bulk_imports/pipeline_spec.rb
@@ -12,6 +12,8 @@ RSpec.describe BulkImports::Pipeline do
klass = Class.new do
include BulkImports::Pipeline
+ abort_on_failure!
+
extractor BulkImports::Extractor, { foo: :bar }
transformer BulkImports::Transformer, { foo: :bar }
loader BulkImports::Loader, { foo: :bar }
@@ -25,6 +27,7 @@ RSpec.describe BulkImports::Pipeline do
expect(BulkImports::MyPipeline.extractors).to contain_exactly({ klass: BulkImports::Extractor, options: { foo: :bar } })
expect(BulkImports::MyPipeline.transformers).to contain_exactly({ klass: BulkImports::Transformer, options: { foo: :bar } })
expect(BulkImports::MyPipeline.loaders).to contain_exactly({ klass: BulkImports::Loader, options: { foo: :bar } })
+ expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true)
end
end
@@ -36,6 +39,7 @@ RSpec.describe BulkImports::Pipeline do
BulkImports::MyPipeline.extractor(klass, options)
BulkImports::MyPipeline.transformer(klass, options)
BulkImports::MyPipeline.loader(klass, options)
+ BulkImports::MyPipeline.abort_on_failure!
expect(BulkImports::MyPipeline.extractors)
.to contain_exactly(
@@ -51,6 +55,8 @@ RSpec.describe BulkImports::Pipeline do
.to contain_exactly(
{ klass: BulkImports::Loader, options: { foo: :bar } },
{ klass: klass, options: options })
+
+ expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true)
end
end
end
diff --git a/spec/models/bulk_imports/failure_spec.rb b/spec/models/bulk_imports/failure_spec.rb
new file mode 100644
index 00000000000..cde62659a48
--- /dev/null
+++ b/spec/models/bulk_imports/failure_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Failure, type: :model do
+ describe 'associations' do
+ it { is_expected.to belong_to(:entity).required }
+ end
+
+ describe 'validations' do
+ before do
+ create(:bulk_import_failure)
+ end
+
+ it { is_expected.to validate_presence_of(:entity) }
+ end
+end
diff --git a/yarn.lock b/yarn.lock
index 5d20efdc21d..e6aeff1ad8a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1173,20 +1173,20 @@
dom-accessibility-api "^0.5.1"
pretty-format "^26.4.2"
-"@toast-ui/editor@^2.5.0":
- version "2.5.0"
- resolved "https://registry.yarnpkg.com/@toast-ui/editor/-/editor-2.5.0.tgz#02779b119eaa6dd7601249d75ca031e0b98400f1"
- integrity sha512-h4LgcGz+oedTqNAaSCp0VpR+k4C6Ag01hdDb1kPvO4aMQ/aTtT8uA34plpmYQgJvM0CWD1mXqWUSPkyJtRzDyA==
+"@toast-ui/editor@^2.5.1":
+ version "2.5.1"
+ resolved "https://registry.yarnpkg.com/@toast-ui/editor/-/editor-2.5.1.tgz#42671c52ca4b97c84f684d09c2966711b36f41a7"
+ integrity sha512-LVNo/YaNItUemEaRFvFAVn7w/0U7yxEheMdn6GEGxqo727rRZD1MH7OTDVq6NeQ+P93VwFpa0i9GGRBhNNEbPQ==
dependencies:
"@types/codemirror" "0.0.71"
codemirror "^5.48.4"
-"@toast-ui/vue-editor@^2.5.0":
- version "2.5.0"
- resolved "https://registry.yarnpkg.com/@toast-ui/vue-editor/-/vue-editor-2.5.0.tgz#8094136588b0f726241b5f89d0754a7169f2ffee"
- integrity sha512-GREAaVOe5esQaQFmFCZLjo6iOtIvqvYhANulvsKpbh4QNnsPLaFRIQoUDSImNPVGkDDQn60wxXBnZVKOl9sMmg==
+"@toast-ui/vue-editor@^2.5.1":
+ version "2.5.1"
+ resolved "https://registry.yarnpkg.com/@toast-ui/vue-editor/-/vue-editor-2.5.1.tgz#0a221d74d5305c8ca20cb11d9eb8ff9206455cfc"
+ integrity sha512-vD0FowDrlMPfR4m1Sd91YthkMLul4lTdiwl1QcDYX+JhIzxXMuQhFABezny/TvKJLxkCkHGpt7XsTjXvMUa04w==
dependencies:
- "@toast-ui/editor" "^2.5.0"
+ "@toast-ui/editor" "^2.5.1"
"@types/aria-query@^4.2.0":
version "4.2.0"