diff options
79 files changed, 1589 insertions, 558 deletions
diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index d4c847cf1c0..86a55e88ab2 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -65,6 +65,7 @@ review-deploy: - deploy || (display_deployment_debug && exit 1) - verify_deploy || exit 1 - disable_sign_ups || (delete_release && exit 1) + - create_sample_projects after_script: # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. # Set DAST_RUN to true when jobs are manually scheduled. diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index 893ae7b93b5..3c563329794 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -20,7 +20,6 @@ If you are only adding documentation, do not add any of the following labels: -- `~"type::feature"` - `~"frontend"` - `~"backend"` - `~"type::bug"` @@ -44,5 +43,5 @@ Documentation-related MRs should be reviewed by a Technical Writer for a non-blo - [ ] Review by assigned maintainer, who can always request/require the above reviews. Maintainer's review can occur before or after a technical writer review. - [ ] Ensure a release milestone is set. -/label ~documentation +/label ~documentation ~"type::maintenance" /assign me diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 70ca8545134..516e0c5f549 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -13.22.0 +13.22.1 diff --git a/app/assets/javascripts/notes/components/notes_app.vue b/app/assets/javascripts/notes/components/notes_app.vue index e568fd0c2d7..c4924cd41f5 100644 --- a/app/assets/javascripts/notes/components/notes_app.vue +++ b/app/assets/javascripts/notes/components/notes_app.vue @@ -104,6 +104,10 @@ export default { }); } + if (this.sortDirDesc) { + return skeletonNotes.concat(this.discussions); + } + return this.discussions.concat(skeletonNotes); }, canReply() { diff --git a/app/assets/stylesheets/framework/files.scss b/app/assets/stylesheets/framework/files.scss index 8f65f349cf9..9209a0c2173 100644 --- a/app/assets/stylesheets/framework/files.scss +++ b/app/assets/stylesheets/framework/files.scss @@ -151,7 +151,17 @@ margin: 0; } + // + // IMPORTANT PERFORMANCE OPTIMIZATION + // + // When viewinng a blame with many commits a lot of content is rendered on the page. + // content-visibility rule below ensure that we only render what is visible to the user, + // thus reducing TBT in the browser. + // Grid is used instead of table layout because content-visibility performs better with it. tr { + content-visibility: auto; + display: grid; + grid-template-columns: 400px max-content auto; border-bottom: 1px solid $gray-darker; &:last-child { @@ -201,6 +211,10 @@ &.lines { padding: 0; } + + .code { + height: 100%; + } } @for $i from 0 through 5 { @@ -222,25 +236,6 @@ color: $gray-900; } } - - // - // IMPORTANT PERFORMANCE OPTIMIZATION - // - // When viewinng a blame with many commits a lot of content is rendered on the page. - // content-visibility rules below ensure that we only render what is visible to the user, thus reducing TBT in the browser. - .commit { - content-visibility: auto; - contain-intrinsic-size: 1px 3em; - } - - code .line { - content-visibility: auto; - contain-intrinsic-size: 1px 1.1875rem; - } - - .line-numbers { - content-visibility: auto; - } } &.logs { diff --git a/app/controllers/clusters/clusters_controller.rb b/app/controllers/clusters/clusters_controller.rb index 32de9e69c85..15a261f572a 100644 --- a/app/controllers/clusters/clusters_controller.rb +++ b/app/controllers/clusters/clusters_controller.rb @@ -280,7 +280,10 @@ class Clusters::ClustersController < Clusters::BaseController end def generate_gcp_authorize_url - state = generate_session_key_redirect(clusterable.new_path(provider: :gcp).to_s) + new_path = clusterable.new_path(provider: :gcp).to_s + error_path = @project ? project_clusters_path(@project) : new_path + + state = generate_session_key_redirect(new_path, error_path) @authorize_url = GoogleApi::CloudPlatform::Client.new( nil, callback_google_api_auth_url, @@ -339,9 +342,10 @@ class Clusters::ClustersController < Clusters::BaseController session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] end - def generate_session_key_redirect(uri) + def generate_session_key_redirect(uri, error_uri) GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key| session[key] = uri + session[:error_uri] = error_uri end end diff --git a/app/controllers/google_api/authorizations_controller.rb b/app/controllers/google_api/authorizations_controller.rb index 76a1c43dfa3..b9c5e87c69c 100644 --- a/app/controllers/google_api/authorizations_controller.rb +++ b/app/controllers/google_api/authorizations_controller.rb @@ -8,19 +8,36 @@ module GoogleApi feature_category :kubernetes_management + ## + # handle the response from google after the user + # goes through authentication and authorization process def callback - token, expires_at = GoogleApi::CloudPlatform::Client - .new(nil, callback_google_api_auth_url) - .get_token(params[:code]) - - session[GoogleApi::CloudPlatform::Client.session_key_for_token] = token - session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] = - expires_at.to_s - + redirect_uri = redirect_uri_from_session + ## + # when the user declines authorizations + # `error` param is returned + if params[:error] + flash[:alert] = _('Google Cloud authorizations required') + redirect_uri = session[:error_uri] + ## + # on success, the `code` param is returned + elsif params[:code] + token, expires_at = GoogleApi::CloudPlatform::Client + .new(nil, callback_google_api_auth_url) + .get_token(params[:code]) + + session[GoogleApi::CloudPlatform::Client.session_key_for_token] = token + session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] = expires_at.to_s + redirect_uri = redirect_uri_from_session + end + ## + # or google may just timeout rescue ::Faraday::TimeoutError, ::Faraday::ConnectionFailed flash[:alert] = _('Timeout connecting to the Google API. Please try again.') + ## + # regardless, we redirect the user appropriately ensure - redirect_to redirect_uri_from_session + redirect_to redirect_uri end private diff --git a/app/controllers/projects/google_cloud/base_controller.rb b/app/controllers/projects/google_cloud/base_controller.rb new file mode 100644 index 00000000000..2a9e89a445b --- /dev/null +++ b/app/controllers/projects/google_cloud/base_controller.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Projects::GoogleCloud::BaseController < Projects::ApplicationController + feature_category :google_cloud + + before_action :admin_project_google_cloud! + before_action :google_oauth2_enabled! + before_action :feature_flag_enabled! + + private + + def admin_project_google_cloud! + access_denied! unless can?(current_user, :admin_project_google_cloud, project) + end + + def google_oauth2_enabled! + config = Gitlab::Auth::OAuth::Provider.config_for('google_oauth2') + if config.app_id.blank? || config.app_secret.blank? + access_denied! 'This GitLab instance not configured for Google Oauth2.' + end + end + + def feature_flag_enabled! + access_denied! unless Feature.enabled?(:incubation_5mp_google_cloud) + end +end diff --git a/app/controllers/projects/google_cloud/service_accounts_controller.rb b/app/controllers/projects/google_cloud/service_accounts_controller.rb new file mode 100644 index 00000000000..21b096a6c66 --- /dev/null +++ b/app/controllers/projects/google_cloud/service_accounts_controller.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +class Projects::GoogleCloud::ServiceAccountsController < Projects::GoogleCloud::BaseController + before_action :validate_gcp_token! + + def index + @google_cloud_path = project_google_cloud_index_path(project) + google_api_client = GoogleApi::CloudPlatform::Client.new(token_in_session, nil) + gcp_projects = google_api_client.list_projects + + if gcp_projects.empty? + @js_data = {}.to_json + render status: :unauthorized, template: 'projects/google_cloud/errors/no_gcp_projects' + else + @js_data = { + gcpProjects: gcp_projects, + environments: project.environments, + cancelPath: project_google_cloud_index_path(project) + }.to_json + end + rescue Google::Apis::ClientError => error + handle_gcp_error(error, project) + end + + def create + google_api_client = GoogleApi::CloudPlatform::Client.new(token_in_session, nil) + service_accounts_service = GoogleCloud::ServiceAccountsService.new(project) + gcp_project = params[:gcp_project] + environment = params[:environment] + generated_name = "GitLab :: #{@project.name} :: #{environment}" + generated_desc = "GitLab generated service account for project '#{@project.name}' and environment '#{environment}'" + + service_account = google_api_client.create_service_account(gcp_project, generated_name, generated_desc) + service_account_key = google_api_client.create_service_account_key(gcp_project, service_account.unique_id) + + service_accounts_service.add_for_project( + environment, + service_account.project_id, + service_account.to_json, + service_account_key.to_json + ) + + redirect_to project_google_cloud_index_path(project), notice: _('Service account generated successfully') + rescue Google::Apis::ClientError, Google::Apis::ServerError, Google::Apis::AuthorizationError => error + handle_gcp_error(error, project) + end + + private + + def validate_gcp_token! + is_token_valid = GoogleApi::CloudPlatform::Client.new(token_in_session, nil) + .validate_token(expires_at_in_session) + + return if is_token_valid + + return_url = project_google_cloud_service_accounts_path(project) + state = generate_session_key_redirect(request.url, return_url) + @authorize_url = GoogleApi::CloudPlatform::Client.new(nil, + callback_google_api_auth_url, + state: state).authorize_url + redirect_to @authorize_url + end + + def generate_session_key_redirect(uri, error_uri) + GoogleApi::CloudPlatform::Client.new_session_key_for_redirect_uri do |key| + session[key] = uri + session[:error_uri] = error_uri + end + end + + def token_in_session + session[GoogleApi::CloudPlatform::Client.session_key_for_token] + end + + def expires_at_in_session + session[GoogleApi::CloudPlatform::Client.session_key_for_expires_at] + end + + def handle_gcp_error(error, project) + Gitlab::ErrorTracking.track_exception(error, project_id: project.id) + @js_data = { error: error.to_s }.to_json + render status: :unauthorized, template: 'projects/google_cloud/errors/gcp_error' + end +end diff --git a/app/controllers/projects/google_cloud_controller.rb b/app/controllers/projects/google_cloud_controller.rb index 7257ed1ef6f..6cc67391d6c 100644 --- a/app/controllers/projects/google_cloud_controller.rb +++ b/app/controllers/projects/google_cloud_controller.rb @@ -1,34 +1,11 @@ # frozen_string_literal: true -class Projects::GoogleCloudController < Projects::ApplicationController - feature_category :google_cloud - - before_action :admin_project_google_cloud? - before_action :google_oauth2_enabled? - before_action :feature_flag_enabled? - +class Projects::GoogleCloudController < Projects::GoogleCloud::BaseController def index @js_data = { serviceAccounts: GoogleCloud::ServiceAccountsService.new(project).find_for_project, - createServiceAccountUrl: '#mocked-url-create-service', + createServiceAccountUrl: project_google_cloud_service_accounts_path(project), emptyIllustrationUrl: ActionController::Base.helpers.image_path('illustrations/pipelines_empty.svg') }.to_json end - - private - - def admin_project_google_cloud? - access_denied! unless can?(current_user, :admin_project_google_cloud, project) - end - - def google_oauth2_enabled? - config = Gitlab::Auth::OAuth::Provider.config_for('google_oauth2') - if config.app_id.blank? || config.app_secret.blank? - access_denied! 'This GitLab instance not configured for Google Oauth2.' - end - end - - def feature_flag_enabled? - access_denied! unless Feature.enabled?(:incubation_5mp_google_cloud) - end end diff --git a/app/helpers/blame_helper.rb b/app/helpers/blame_helper.rb index 82c74e2416d..5117f7c6d9c 100644 --- a/app/helpers/blame_helper.rb +++ b/app/helpers/blame_helper.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true module BlameHelper + BODY_FONT_SIZE = "0.875rem" + COMMIT_LINE_HEIGHT = 3 # 150% * 2 lines of text + COMMIT_PADDING = "10px" # 5px from both top and bottom + COMMIT_BLOCK_HEIGHT_EXP = "(#{BODY_FONT_SIZE} * #{COMMIT_LINE_HEIGHT}) + #{COMMIT_PADDING}" + CODE_LINE_HEIGHT = 1.1875 + CODE_PADDING = "20px" # 10px from both top and bottom + def age_map_duration(blame_groups, project) now = Time.zone.now start_date = blame_groups.map { |blame_group| blame_group[:commit].committed_date } @@ -24,4 +31,12 @@ module BlameHelper "blame-commit-age-#{age_group}" end end + + def intrinsic_row_css(line_count) + # using rems here because the size of the row depends on the text size + # which can be customized via user agent styles and browser preferences + total_line_height_exp = "#{line_count * CODE_LINE_HEIGHT}rem + #{CODE_PADDING}" + row_height_exp = line_count == 1 ? COMMIT_BLOCK_HEIGHT_EXP : total_line_height_exp + "contain-intrinsic-size: 1px calc(#{row_height_exp})" + end end diff --git a/app/helpers/tab_helper.rb b/app/helpers/tab_helper.rb index e53e35baac3..d503011e9c1 100644 --- a/app/helpers/tab_helper.rb +++ b/app/helpers/tab_helper.rb @@ -63,6 +63,19 @@ module TabHelper end end + # Creates a <gl-badge> for use inside tabs. + # + # html_options - The html_options hash (default: {}) + def gl_tab_counter_badge(count, html_options = {}) + gl_badge_tag( + count, + { size: :sm }, + html_options.merge( + class: ['gl-tab-counter-badge', *html_options[:class]] + ) + ) + end + # Navigation link helper # # Returns an `li` element with an 'active' class if the supplied @@ -211,12 +224,3 @@ module TabHelper current_page?(options) end end - -def gl_tab_counter_badge(count, html_options = {}) - badge_classes = %w[badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge] - content_tag(:span, - count, - class: [*html_options[:class], badge_classes].join(' '), - data: html_options[:data] - ) -end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 2368be6196c..159617bc0a9 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -104,8 +104,12 @@ class BulkImports::Entity < ApplicationRecord end end + def entity_type + source_type.gsub('_entity', '') + end + def pluralized_name - source_type.gsub('_entity', '').pluralize + entity_type.pluralize end def export_relations_url_path @@ -116,6 +120,14 @@ class BulkImports::Entity < ApplicationRecord "#{export_relations_url_path}/download?relation=#{relation}" end + def project? + source_type == 'project_entity' + end + + def group? + source_type == 'group_entity' + end + private def validate_parent_is_a_group diff --git a/app/models/issue.rb b/app/models/issue.rb index 47dc084d69c..581f7c18277 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -63,6 +63,7 @@ class Issue < ApplicationRecord has_many :issue_assignees has_many :issue_email_participants + has_one :email has_many :assignees, class_name: "User", through: :issue_assignees has_many :zoom_meetings has_many :user_mentions, class_name: "IssueUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/issue/email.rb b/app/models/issue/email.rb new file mode 100644 index 00000000000..730fda5cdb4 --- /dev/null +++ b/app/models/issue/email.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class Issue::Email < ApplicationRecord + self.table_name = 'issue_emails' + + belongs_to :issue + + validates :email_message_id, uniqueness: true, presence: true, length: { maximum: 1000 } + validates :issue, presence: true, uniqueness: true +end diff --git a/app/services/google_cloud/service_accounts_service.rb b/app/services/google_cloud/service_accounts_service.rb index 29ed69693b0..a512b27493d 100644 --- a/app/services/google_cloud/service_accounts_service.rb +++ b/app/services/google_cloud/service_accounts_service.rb @@ -27,6 +27,24 @@ module GoogleCloud end end + def add_for_project(environment, gcp_project_id, service_account, service_account_key) + project_var_create_or_replace( + environment, + 'GCP_PROJECT_ID', + gcp_project_id + ) + project_var_create_or_replace( + environment, + 'GCP_SERVICE_ACCOUNT', + service_account + ) + project_var_create_or_replace( + environment, + 'GCP_SERVICE_ACCOUNT_KEY', + service_account_key + ) + end + private def group_vars_by_environment @@ -36,5 +54,12 @@ module GoogleCloud grouped[variable.environment_scope][variable.key] = variable.value end end + + def project_var_create_or_replace(environment_scope, key, value) + params = { key: key, filter: { environment_scope: environment_scope } } + existing_variable = ::Ci::VariablesFinder.new(@project, params).execute.first + existing_variable.destroy if existing_variable + @project.variables.create!(key: key, value: value, environment_scope: environment_scope, protected: true) + end end end diff --git a/app/views/projects/blame/show.html.haml b/app/views/projects/blame/show.html.haml index 704576619a7..ae8f89bf16a 100644 --- a/app/views/projects/blame/show.html.haml +++ b/app/views/projects/blame/show.html.haml @@ -27,7 +27,7 @@ - commit_data = @blame.commit_data(blame_group[:commit]) - line_count = blame_group[:lines].count - %tr + %tr{ style: intrinsic_row_css(line_count) } %td.blame-commit{ class: commit_data.age_map_class } .commit = commit_data.author_avatar diff --git a/app/views/projects/google_cloud/errors/gcp_error.html.haml b/app/views/projects/google_cloud/errors/gcp_error.html.haml new file mode 100644 index 00000000000..b91a85250b3 --- /dev/null +++ b/app/views/projects/google_cloud/errors/gcp_error.html.haml @@ -0,0 +1,6 @@ +- breadcrumb_title _('Google Cloud') +- page_title _('Google Cloud') + +- @content_class = "limit-container-width" unless fluid_layout + +#js-google-cloud-error-gcp-error{ data: @js_data } diff --git a/app/views/projects/google_cloud/errors/no_gcp_projects.html.haml b/app/views/projects/google_cloud/errors/no_gcp_projects.html.haml new file mode 100644 index 00000000000..743b757de57 --- /dev/null +++ b/app/views/projects/google_cloud/errors/no_gcp_projects.html.haml @@ -0,0 +1,6 @@ +- breadcrumb_title _('Google Cloud') +- page_title _('Google Cloud') + +- @content_class = "limit-container-width" unless fluid_layout + +#js-google-cloud-error-no-gcp-projects{ data: @js_data } diff --git a/app/views/projects/google_cloud/service_accounts/index.html.haml b/app/views/projects/google_cloud/service_accounts/index.html.haml new file mode 100644 index 00000000000..69b2123d723 --- /dev/null +++ b/app/views/projects/google_cloud/service_accounts/index.html.haml @@ -0,0 +1,8 @@ +- add_to_breadcrumbs _('Google Cloud'), @google_cloud_path +- breadcrumb_title _('Service Account') +- page_title _('Service Account') + +- @content_class = "limit-container-width" unless fluid_layout + += form_tag project_google_cloud_service_accounts_path(@project), method: 'post' do + #js-google-cloud-service-accounts{ data: @js_data } diff --git a/config/routes/project.rb b/config/routes/project.rb index 8a8668a2314..5f1b35d67c0 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -317,6 +317,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resources :google_cloud, only: [:index] + namespace :google_cloud do + resources :service_accounts, only: [:index, :create] + end + resources :environments, except: [:destroy] do member do post :stop diff --git a/db/migrate/20211110092710_create_issue_emails.rb b/db/migrate/20211110092710_create_issue_emails.rb new file mode 100644 index 00000000000..5f6104fa2c3 --- /dev/null +++ b/db/migrate/20211110092710_create_issue_emails.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateIssueEmails < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def up + create_table :issue_emails do |t| + t.references :issue, index: true, null: false, unique: true, foreign_key: { on_delete: :cascade } + t.text :email_message_id, null: false, limit: 1000 + + t.index :email_message_id + end + end + + def down + drop_table :issue_emails + end +end diff --git a/db/schema_migrations/20211110092710 b/db/schema_migrations/20211110092710 new file mode 100644 index 00000000000..691194456d4 --- /dev/null +++ b/db/schema_migrations/20211110092710 @@ -0,0 +1 @@ +f6312d56d2ac77537383c8671d73ad202fed9bb8eddba4bdb24d19dbe821cdf3
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index d323b988a75..3db0f0c14e9 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -15278,6 +15278,22 @@ CREATE SEQUENCE issue_email_participants_id_seq ALTER SEQUENCE issue_email_participants_id_seq OWNED BY issue_email_participants.id; +CREATE TABLE issue_emails ( + id bigint NOT NULL, + issue_id bigint NOT NULL, + email_message_id text NOT NULL, + CONSTRAINT check_5abf3e6aea CHECK ((char_length(email_message_id) <= 1000)) +); + +CREATE SEQUENCE issue_emails_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE issue_emails_id_seq OWNED BY issue_emails.id; + CREATE TABLE issue_links ( id integer NOT NULL, source_id integer NOT NULL, @@ -21565,6 +21581,8 @@ ALTER TABLE ONLY issue_customer_relations_contacts ALTER COLUMN id SET DEFAULT n ALTER TABLE ONLY issue_email_participants ALTER COLUMN id SET DEFAULT nextval('issue_email_participants_id_seq'::regclass); +ALTER TABLE ONLY issue_emails ALTER COLUMN id SET DEFAULT nextval('issue_emails_id_seq'::regclass); + ALTER TABLE ONLY issue_links ALTER COLUMN id SET DEFAULT nextval('issue_links_id_seq'::regclass); ALTER TABLE ONLY issue_metrics ALTER COLUMN id SET DEFAULT nextval('issue_metrics_id_seq'::regclass); @@ -23238,6 +23256,9 @@ ALTER TABLE ONLY issue_customer_relations_contacts ALTER TABLE ONLY issue_email_participants ADD CONSTRAINT issue_email_participants_pkey PRIMARY KEY (id); +ALTER TABLE ONLY issue_emails + ADD CONSTRAINT issue_emails_pkey PRIMARY KEY (id); + ALTER TABLE ONLY issue_links ADD CONSTRAINT issue_links_pkey PRIMARY KEY (id); @@ -26159,6 +26180,10 @@ CREATE INDEX index_issue_customer_relations_contacts_on_contact_id ON issue_cust CREATE UNIQUE INDEX index_issue_email_participants_on_issue_id_and_lower_email ON issue_email_participants USING btree (issue_id, lower(email)); +CREATE INDEX index_issue_emails_on_email_message_id ON issue_emails USING btree (email_message_id); + +CREATE INDEX index_issue_emails_on_issue_id ON issue_emails USING btree (issue_id); + CREATE INDEX index_issue_links_on_source_id ON issue_links USING btree (source_id); CREATE UNIQUE INDEX index_issue_links_on_source_id_and_target_id ON issue_links USING btree (source_id, target_id); @@ -30992,6 +31017,9 @@ ALTER TABLE ONLY packages_packages ALTER TABLE ONLY cluster_platforms_kubernetes ADD CONSTRAINT fk_rails_e1e2cf841a FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; +ALTER TABLE ONLY issue_emails + ADD CONSTRAINT fk_rails_e2ee00a8f7 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_finding_evidences ADD CONSTRAINT fk_rails_e3205a0c65 FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE; diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index a213c936be1..21be553fb0d 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -196,7 +196,7 @@ successfully, you must replicate their data using some other means. |[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | No | Designs also require replication of LFS objects and Uploads. | |[Package Registry](../../../user/packages/package_registry/index.md) | **Yes** (13.2) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.10) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_package_file_replication`, enabled by default. | |[Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (13.12) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_terraform_state_version_replication`, enabled by default. Verification was behind the feature flag `geo_terraform_state_version_verification`, which was removed in 14.0| -|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.5) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification is behind the feature flag `geo_merge_request_diff_verification`, enabled by default in 14.5.| +|[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification is behind the feature flag `geo_merge_request_diff_verification`, enabled by default in 14.6.| |[Versioned snippets](../../../user/snippets.md#versioned-snippets) | [**Yes** (13.7)](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [**Yes** (14.2)](https://gitlab.com/groups/gitlab-org/-/epics/2810) | No | Verification was implemented behind the feature flag `geo_snippet_repository_verification` in 13.11, and the feature flag was removed in 14.2. | |[GitLab Pages](../../pages/index.md) | [**Yes** (14.3)](https://gitlab.com/groups/gitlab-org/-/epics/589) | No | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_pages_deployment_replication`, enabled by default. | |[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. | diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 1a23d175dbf..b5f27f6d1f4 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -255,7 +255,7 @@ this, you must point Praefect to PgBouncer by setting Praefect database paramete ```ruby praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 +praefect['database_port'] = 5432 praefect['database_user'] = 'praefect' praefect['database_password'] = PRAEFECT_SQL_PASSWORD praefect['database_dbname'] = 'praefect_production' diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 9c3c33e1fa8..d42e76248d9 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 10k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 card "**Internal Load Balancer**" as ilb #9370DB @@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster { card "Database" as database { collections "**PGBouncer** x3" as pgbouncer #4EA7FF - card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF - collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF + collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF pgbouncer -[#4EA7FF]-> postgres_primary postgres_primary .[#4EA7FF]> postgres_secondary @@ -83,31 +85,38 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor +elb -[#6a9be7,norank]--> monitor -gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis +gitlab -[#32CD32,norank]--> ilb +gitlab -[#32CD32]r-> object_storage +gitlab -[#32CD32]----> redis +gitlab .[#32CD32]----> database gitlab -[hidden]-> monitor gitlab -[hidden]-> consul -sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis +sidekiq -[#ff8dd1,norank]--> ilb +sidekiq -[#ff8dd1]r-> object_storage +sidekiq -[#ff8dd1]----> redis +sidekiq .[#ff8dd1]----> database sidekiq -[hidden]-> monitor sidekiq -[hidden]-> consul -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden]--> redis +ilb -[hidden]u-> consul +ilb -[hidden]u-> monitor consul .[#e76a9b]u-> gitlab consul .[#e76a9b]u-> sidekiq -consul .[#e76a9b]> monitor +consul .[#e76a9b]r-> monitor consul .[#e76a9b]-> database consul .[#e76a9b]-> gitaly_cluster consul .[#e76a9b,norank]--> redis @@ -471,8 +480,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s ## Configure PostgreSQL -In this section, you'll be guided through configuring an external PostgreSQL database -to be used with GitLab. +In this section, you'll be guided through configuring a highly available PostgreSQL +cluster to be used with GitLab. ### Provide your own PostgreSQL instance @@ -488,12 +497,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL: needs privileges to create the `gitlabhq_production` database. 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). +1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md) + with multiple read replicas is recommended. See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab +The recommended Omnibus GitLab configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three PostgreSQL nodes. +- A minimum of three Consul server nodes. +- A minimum of three PgBouncer nodes that track and handle primary database reads and writes. + - An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes. +- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled. + + A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary. + The following IPs will be used as an example: - `10.6.0.21`: PostgreSQL primary @@ -548,8 +570,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except Patroni and Consul - roles(['patroni_role']) + # Disable all components except Patroni, PgBouncer and Consul + roles(['patroni_role', 'pgbouncer_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -594,6 +616,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace 10.6.0.0/24 with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32) + # Local PgBouncer service for Database Load Balancing + pgbouncer['databases'] = { + gitlabhq_production: { + host: "127.0.0.1", + user: "pgbouncer", + password: '<pgbouncer_password_hash>' + } + } + # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -654,9 +685,11 @@ If the 'State' column for any node doesn't say "running", check the </a> </div> -## Configure PgBouncer +### Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer +for tracking and handling reads/writes to the primary database. -Now that the PostgreSQL servers are all set up, let's configure PgBouncer. The following IPs will be used as an example: - `10.6.0.31`: PgBouncer 1 @@ -1671,8 +1704,8 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1797,6 +1830,8 @@ On each node perform the following: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -2212,6 +2247,7 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 10k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 @@ -2221,7 +2257,6 @@ card "Kubernetes via Helm Charts" as kubernetes { collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Prometheus + Grafana**" as monitor #7FFFD4 card "**Supporting Services**" as support } @@ -2249,37 +2284,33 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]-> monitor +elb -[hidden]-> sidekiq elb -[hidden]-> support gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis -gitlab -[hidden]--> consul +gitlab -[#32CD32]r--> object_storage +gitlab -[#32CD32,norank]----> redis +gitlab -[#32CD32]----> database sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis -sidekiq -[hidden]--> consul - -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +sidekiq -[#ff8dd1]r--> object_storage +sidekiq -[#ff8dd1,norank]----> redis +sidekiq .[#ff8dd1]----> database -consul .[#e76a9b]-> database -consul .[#e76a9b]-> gitaly_cluster -consul .[#e76a9b,norank]--> redis +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden,norank]--> redis -monitor .[#7FFFD4]> consul -monitor .[#7FFFD4]-> database -monitor .[#7FFFD4]-> gitaly_cluster -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4]> ilb -monitor .[#7FFFD4,norank]u--> elb +consul .[#e76a9b]--> database +consul .[#e76a9b,norank]--> gitaly_cluster +consul .[#e76a9b]--> redis @enduml ``` diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 25cafbe667b..95892dcd188 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 25k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 card "**Internal Load Balancer**" as ilb #9370DB @@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster { card "Database" as database { collections "**PGBouncer** x3" as pgbouncer #4EA7FF - card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF - collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF + collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF pgbouncer -[#4EA7FF]-> postgres_primary postgres_primary .[#4EA7FF]> postgres_secondary @@ -83,31 +85,38 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor +elb -[#6a9be7,norank]--> monitor -gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis +gitlab -[#32CD32,norank]--> ilb +gitlab -[#32CD32]r-> object_storage +gitlab -[#32CD32]----> redis +gitlab .[#32CD32]----> database gitlab -[hidden]-> monitor gitlab -[hidden]-> consul -sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis +sidekiq -[#ff8dd1,norank]--> ilb +sidekiq -[#ff8dd1]r-> object_storage +sidekiq -[#ff8dd1]----> redis +sidekiq .[#ff8dd1]----> database sidekiq -[hidden]-> monitor sidekiq -[hidden]-> consul -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden]--> redis +ilb -[hidden]u-> consul +ilb -[hidden]u-> monitor consul .[#e76a9b]u-> gitlab consul .[#e76a9b]u-> sidekiq -consul .[#e76a9b]> monitor +consul .[#e76a9b]r-> monitor consul .[#e76a9b]-> database consul .[#e76a9b]-> gitaly_cluster consul .[#e76a9b,norank]--> redis @@ -474,8 +483,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s ## Configure PostgreSQL -In this section, you'll be guided through configuring an external PostgreSQL database -to be used with GitLab. +In this section, you'll be guided through configuring a highly available PostgreSQL +cluster to be used with GitLab. ### Provide your own PostgreSQL instance @@ -491,12 +500,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL: needs privileges to create the `gitlabhq_production` database. 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). +1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md) + with multiple read replicas is recommended. See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab +The recommended Omnibus GitLab configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three PostgreSQL nodes. +- A minimum of three Consul server nodes. +- A minimum of three PgBouncer nodes that track and handle primary database reads and writes. + - An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes. +- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled. + + A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary. + The following IPs will be used as an example: - `10.6.0.21`: PostgreSQL primary @@ -551,8 +573,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except Patroni and Consul - roles(['patroni_role']) + # Disable all components except Patroni, PgBouncer and Consul + roles(['patroni_role', 'pgbouncer_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -597,6 +619,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace 10.6.0.0/24 with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32) + # Local PgBouncer service for Database Load Balancing + pgbouncer['databases'] = { + gitlabhq_production: { + host: "127.0.0.1", + user: "pgbouncer", + password: '<pgbouncer_password_hash>' + } + } + # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -657,9 +688,11 @@ If the 'State' column for any node doesn't say "running", check the </a> </div> -## Configure PgBouncer +### Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer +for tracking and handling reads/writes to the primary database. -Now that the PostgreSQL servers are all set up, let's configure PgBouncer. The following IPs will be used as an example: - `10.6.0.31`: PgBouncer 1 @@ -1677,8 +1710,8 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1805,6 +1838,8 @@ On each node perform the following: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -2212,16 +2247,16 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 25k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x7" as gitlab #32CD32 + collections "**Webservice** x4" as gitlab #32CD32 collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Prometheus + Grafana**" as monitor #7FFFD4 card "**Supporting Services**" as support } @@ -2249,37 +2284,33 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]-> monitor +elb -[hidden]-> sidekiq elb -[hidden]-> support gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis -gitlab -[hidden]--> consul +gitlab -[#32CD32]r--> object_storage +gitlab -[#32CD32,norank]----> redis +gitlab -[#32CD32]----> database sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis -sidekiq -[hidden]--> consul - -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +sidekiq -[#ff8dd1]r--> object_storage +sidekiq -[#ff8dd1,norank]----> redis +sidekiq .[#ff8dd1]----> database -consul .[#e76a9b]-> database -consul .[#e76a9b]-> gitaly_cluster -consul .[#e76a9b,norank]--> redis +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden,norank]--> redis -monitor .[#7FFFD4]> consul -monitor .[#7FFFD4]-> database -monitor .[#7FFFD4]-> gitaly_cluster -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4]> ilb -monitor .[#7FFFD4,norank]u--> elb +consul .[#e76a9b]--> database +consul .[#e76a9b,norank]--> gitaly_cluster +consul .[#e76a9b]--> redis @enduml ``` diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index e619294704f..be982712c89 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -41,6 +41,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 2k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 collections "**GitLab Rails** x3" as gitlab #32CD32 @@ -1038,6 +1040,7 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 2k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 @@ -1045,10 +1048,8 @@ card "Kubernetes via Helm Charts" as kubernetes { together { collections "**Webservice** x3" as gitlab #32CD32 collections "**Sidekiq** x2" as sidekiq #ff8dd1 + card "**Supporting Services**" as support } - - card "**Prometheus + Grafana**" as monitor #7FFFD4 - card "**Supporting Services**" as support } card "**Gitaly**" as gitaly #FF8C00 @@ -1057,7 +1058,6 @@ card "**Redis**" as redis #FF6347 cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor gitlab -[#32CD32]--> gitaly gitlab -[#32CD32]--> postgres @@ -1066,14 +1066,8 @@ gitlab -[#32CD32]--> redis sidekiq -[#ff8dd1]--> gitaly sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> postgres -sidekiq -[#ff8dd1]---> redis - -monitor .[#7FFFD4]u-> gitlab -monitor .[#7FFFD4]-> gitaly -monitor .[#7FFFD4]-> postgres -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4,norank]u--> elb +sidekiq -[#ff8dd1]--> postgres +sidekiq -[#ff8dd1]--> redis @enduml ``` diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 9332ae8d271..89ad619b048 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -58,6 +58,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 3k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 card "**Internal Load Balancer**" as ilb #9370DB @@ -66,7 +68,10 @@ together { collections "**Sidekiq** x4" as sidekiq #ff8dd1 } -card "**Prometheus + Grafana**" as monitor #7FFFD4 +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} card "Gitaly Cluster" as gitaly_cluster { collections "**Praefect** x3" as praefect #FF8C00 @@ -79,47 +84,45 @@ card "Gitaly Cluster" as gitaly_cluster { card "Database" as database { collections "**PGBouncer** x3" as pgbouncer #4EA7FF - card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF - collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF + collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF pgbouncer -[#4EA7FF]-> postgres_primary postgres_primary .[#4EA7FF]> postgres_secondary } -card "**Consul + Sentinel**" as consul_sentinel { - collections "**Consul** x3" as consul #e76a9b - collections "**Redis Sentinel** x3" as sentinel #e6e727 -} - card "Redis" as redis { collections "**Redis** x3" as redis_nodes #FF6347 - - redis_nodes <.[#FF6347]- sentinel } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor +elb -[#6a9be7,norank]--> monitor -gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis +gitlab -[#32CD32,norank]--> ilb +gitlab -[#32CD32]r-> object_storage +gitlab -[#32CD32]----> redis +gitlab .[#32CD32]----> database gitlab -[hidden]-> monitor gitlab -[hidden]-> consul -sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis +sidekiq -[#ff8dd1,norank]--> ilb +sidekiq -[#ff8dd1]r-> object_storage +sidekiq -[#ff8dd1]----> redis +sidekiq .[#ff8dd1]----> database sidekiq -[hidden]-> monitor sidekiq -[hidden]-> consul -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden]--> redis +ilb -[hidden]u-> consul +ilb -[hidden]u-> monitor consul .[#e76a9b]u-> gitlab consul .[#e76a9b]u-> sidekiq -consul .[#e76a9b]> monitor +consul .[#e76a9b]r-> monitor consul .[#e76a9b]-> database consul .[#e76a9b]-> gitaly_cluster consul .[#e76a9b,norank]--> redis @@ -769,8 +772,8 @@ run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s ## Configure PostgreSQL -In this section, you'll be guided through configuring an external PostgreSQL database -to be used with GitLab. +In this section, you'll be guided through configuring a highly available PostgreSQL +cluster to be used with GitLab. ### Provide your own PostgreSQL instance @@ -786,12 +789,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL: needs privileges to create the `gitlabhq_production` database. 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). +1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md) + with multiple read replicas is recommended. See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab +The recommended Omnibus GitLab configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three PostgreSQL nodes. +- A minimum of three Consul server nodes. +- A minimum of three PgBouncer nodes that track and handle primary database reads and writes. + - An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes. +- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled. + + A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary. + The following IPs will be used as an example: - `10.6.0.31`: PostgreSQL primary @@ -846,8 +862,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except Patroni and Consul - roles(['patroni_role']) + # Disable all components except Patroni, PgBouncer and Consul + roles(['patroni_role', 'pgbouncer_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -892,6 +908,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace 10.6.0.0/24 with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32) + # Local PgBouncer service for Database Load Balancing + pgbouncer['databases'] = { + gitlabhq_production: { + host: "127.0.0.1", + user: "pgbouncer", + password: '<pgbouncer_password_hash>' + } + } + # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -952,9 +977,11 @@ If the 'State' column for any node doesn't say "running", check the </a> </div> -## Configure PgBouncer +### Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer +for tracking and handling reads/writes to the primary database. -Now that the PostgreSQL servers are all set up, let's configure PgBouncer. The following IPs will be used as an example: - `10.6.0.21`: PgBouncer 1 @@ -1613,8 +1640,8 @@ To configure the Sidekiq nodes, one each one: gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1773,6 +1800,8 @@ On each node perform the following: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -2183,25 +2212,21 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 3k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x2" as gitlab #32CD32 - collections "**Sidekiq** x3" as sidekiq #ff8dd1 + collections "**Webservice** x4" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Prometheus + Grafana**" as monitor #7FFFD4 card "**Supporting Services**" as support } card "**Internal Load Balancer**" as ilb #9370DB - -card "**Consul + Sentinel**" as consul_sentinel { - collections "**Consul** x3" as consul #e76a9b - collections "**Redis Sentinel** x3" as sentinel #e6e727 -} +collections "**Consul** x3" as consul #e76a9b card "Gitaly Cluster" as gitaly_cluster { collections "**Praefect** x3" as praefect #FF8C00 @@ -2221,41 +2246,33 @@ card "Database" as database { postgres_primary .[#4EA7FF]> postgres_secondary } -card "Redis" as redis { +card "redis" as redis { collections "**Redis** x3" as redis_nodes #FF6347 - - redis_nodes <.[#FF6347]- sentinel } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]-> monitor +elb -[hidden]-> sidekiq elb -[hidden]-> support gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis -gitlab -[hidden]--> consul +gitlab -[#32CD32]r--> object_storage +gitlab -[#32CD32,norank]----> redis +gitlab -[#32CD32]----> database sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis -sidekiq -[hidden]--> consul - -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +sidekiq -[#ff8dd1]r--> object_storage +sidekiq -[#ff8dd1,norank]----> redis +sidekiq .[#ff8dd1]----> database -consul .[#e76a9b]-> database -consul .[#e76a9b]-> gitaly_cluster -consul .[#e76a9b,norank]--> redis +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden,norank]--> redis -monitor .[#7FFFD4]> consul -monitor .[#7FFFD4]-> database -monitor .[#7FFFD4]-> gitaly_cluster -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4]> ilb -monitor .[#7FFFD4,norank]u--> elb +consul .[#e76a9b]--> database +consul .[#e76a9b,norank]--> gitaly_cluster +consul .[#e76a9b]--> redis @enduml ``` diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index bbdf798d9ad..0fc6073f754 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -49,6 +49,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 50k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 card "**Internal Load Balancer**" as ilb #9370DB @@ -73,8 +75,8 @@ card "Gitaly Cluster" as gitaly_cluster { card "Database" as database { collections "**PGBouncer** x3" as pgbouncer #4EA7FF - card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF - collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF + collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF pgbouncer -[#4EA7FF]-> postgres_primary postgres_primary .[#4EA7FF]> postgres_secondary @@ -83,31 +85,38 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor +elb -[#6a9be7,norank]--> monitor -gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis +gitlab -[#32CD32,norank]--> ilb +gitlab -[#32CD32]r-> object_storage +gitlab -[#32CD32]----> redis +gitlab .[#32CD32]----> database gitlab -[hidden]-> monitor gitlab -[hidden]-> consul -sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis +sidekiq -[#ff8dd1,norank]--> ilb +sidekiq -[#ff8dd1]r-> object_storage +sidekiq -[#ff8dd1]----> redis +sidekiq .[#ff8dd1]----> database sidekiq -[hidden]-> monitor sidekiq -[hidden]-> consul -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden]--> redis +ilb -[hidden]u-> consul +ilb -[hidden]u-> monitor consul .[#e76a9b]u-> gitlab consul .[#e76a9b]u-> sidekiq -consul .[#e76a9b]> monitor +consul .[#e76a9b]r-> monitor consul .[#e76a9b]-> database consul .[#e76a9b]-> gitaly_cluster consul .[#e76a9b,norank]--> redis @@ -480,8 +489,8 @@ run: node-exporter: (pid 30093) 76833s; run: log: (pid 29663) 76855s ## Configure PostgreSQL -In this section, you'll be guided through configuring an external PostgreSQL database -to be used with GitLab. +In this section, you'll be guided through configuring a highly available PostgreSQL +cluster to be used with GitLab. ### Provide your own PostgreSQL instance @@ -497,12 +506,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL: needs privileges to create the `gitlabhq_production` database. 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). +1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md) + with multiple read replicas is recommended. See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab +The recommended Omnibus GitLab configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three PostgreSQL nodes. +- A minimum of three Consul server nodes. +- A minimum of three PgBouncer nodes that track and handle primary database reads and writes. + - An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes. +- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled. + + A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary. + The following IPs will be used as an example: - `10.6.0.21`: PostgreSQL primary @@ -557,8 +579,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except Patroni and Consul - roles(['patroni_role']) + # Disable all components except Patroni, PgBouncer and Consul + roles(['patroni_role', 'pgbouncer_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -604,6 +626,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace 10.6.0.0/24 with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32) + # Local PgBouncer service for Database Load Balancing + pgbouncer['databases'] = { + gitlabhq_production: { + host: "127.0.0.1", + user: "pgbouncer", + password: '<pgbouncer_password_hash>' + } + } + # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -664,9 +695,11 @@ If the 'State' column for any node doesn't say "running", check the </a> </div> -## Configure PgBouncer +### Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer +for tracking and handling reads/writes to the primary database. -Now that the PostgreSQL servers are all set up, let's configure PgBouncer. The following IPs will be used as an example: - `10.6.0.31`: PgBouncer 1 @@ -891,7 +924,7 @@ a node and change its status from primary to replica (and vice versa). package of your choice. Be sure to both follow _only_ installation steps 1 and 2 on the page, and to select the correct Omnibus GitLab package, with the same version and type (Community or Enterprise editions) as your current install. -1. Edit `/etc/gitlab/gitlab.rb` and add the same contents as the priimary node in the previous section by replacing `redis_master_node` with `redis_replica_node`: +1. Edit `/etc/gitlab/gitlab.rb` and add the same contents as the primary node in the previous section by replacing `redis_master_node` with `redis_replica_node`: ```ruby # Specify server role as 'redis_replica_role' with Sentinel and enable Consul agent @@ -1684,8 +1717,8 @@ To configure the Sidekiq nodes, on each one: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1819,6 +1852,8 @@ On each node perform the following: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.21', '10.6.0.22', '10.6.0.23'] } # PostgreSQL IPs + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -2226,16 +2261,16 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 50k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x16" as gitlab #32CD32 + collections "**Webservice** x4" as gitlab #32CD32 collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Prometheus + Grafana**" as monitor #7FFFD4 card "**Supporting Services**" as support } @@ -2263,37 +2298,33 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 + + redis_cache -[hidden]-> redis_persistent } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]-> monitor +elb -[hidden]-> sidekiq elb -[hidden]-> support gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis -gitlab -[hidden]--> consul +gitlab -[#32CD32]r--> object_storage +gitlab -[#32CD32,norank]----> redis +gitlab -[#32CD32]----> database sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis -sidekiq -[hidden]--> consul - -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +sidekiq -[#ff8dd1]r--> object_storage +sidekiq -[#ff8dd1,norank]----> redis +sidekiq .[#ff8dd1]----> database -consul .[#e76a9b]-> database -consul .[#e76a9b]-> gitaly_cluster -consul .[#e76a9b,norank]--> redis +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden,norank]--> redis -monitor .[#7FFFD4]> consul -monitor .[#7FFFD4]-> database -monitor .[#7FFFD4]-> gitaly_cluster -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4]> ilb -monitor .[#7FFFD4,norank]u--> elb +consul .[#e76a9b]--> database +consul .[#e76a9b,norank]--> gitaly_cluster +consul .[#e76a9b]--> redis @enduml ``` diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index a1921f50e4e..4f0b333f2b0 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -55,6 +55,8 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 5k +skinparam linetype ortho + card "**External Load Balancer**" as elb #6a9be7 card "**Internal Load Balancer**" as ilb #9370DB @@ -63,7 +65,10 @@ together { collections "**Sidekiq** x4" as sidekiq #ff8dd1 } -card "**Prometheus + Grafana**" as monitor #7FFFD4 +together { + card "**Prometheus + Grafana**" as monitor #7FFFD4 + collections "**Consul** x3" as consul #e76a9b +} card "Gitaly Cluster" as gitaly_cluster { collections "**Praefect** x3" as praefect #FF8C00 @@ -76,47 +81,45 @@ card "Gitaly Cluster" as gitaly_cluster { card "Database" as database { collections "**PGBouncer** x3" as pgbouncer #4EA7FF - card "**PostgreSQL** (Primary)" as postgres_primary #4EA7FF - collections "**PostgreSQL** (Secondary) x2" as postgres_secondary #4EA7FF + card "**PostgreSQL** //Primary//" as postgres_primary #4EA7FF + collections "**PostgreSQL** //Secondary// x2" as postgres_secondary #4EA7FF pgbouncer -[#4EA7FF]-> postgres_primary postgres_primary .[#4EA7FF]> postgres_secondary } -card "**Consul + Sentinel**" as consul_sentinel { - collections "**Consul** x3" as consul #e76a9b - collections "**Redis Sentinel** x3" as sentinel #e6e727 -} - card "Redis" as redis { collections "**Redis** x3" as redis_nodes #FF6347 - - redis_nodes <.[#FF6347]- sentinel } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]--> monitor +elb -[#6a9be7,norank]--> monitor -gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis +gitlab -[#32CD32,norank]--> ilb +gitlab -[#32CD32]r-> object_storage +gitlab -[#32CD32]----> redis +gitlab .[#32CD32]----> database gitlab -[hidden]-> monitor gitlab -[hidden]-> consul -sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis +sidekiq -[#ff8dd1,norank]--> ilb +sidekiq -[#ff8dd1]r-> object_storage +sidekiq -[#ff8dd1]----> redis +sidekiq .[#ff8dd1]----> database sidekiq -[hidden]-> monitor sidekiq -[hidden]-> consul -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden]--> redis +ilb -[hidden]u-> consul +ilb -[hidden]u-> monitor consul .[#e76a9b]u-> gitlab consul .[#e76a9b]u-> sidekiq -consul .[#e76a9b]> monitor +consul .[#e76a9b]r-> monitor consul .[#e76a9b]-> database consul .[#e76a9b]-> gitaly_cluster consul .[#e76a9b,norank]--> redis @@ -760,8 +763,8 @@ run: sentinel: (pid 30098) 76832s; run: log: (pid 29704) 76850s ## Configure PostgreSQL -In this section, you'll be guided through configuring an external PostgreSQL database -to be used with GitLab. +In this section, you'll be guided through configuring a highly available PostgreSQL +cluster to be used with GitLab. ### Provide your own PostgreSQL instance @@ -777,12 +780,25 @@ If you use a cloud-managed service, or provide your own PostgreSQL: needs privileges to create the `gitlabhq_production` database. 1. Configure the GitLab application servers with the appropriate details. This step is covered in [Configuring the GitLab Rails application](#configure-gitlab-rails). +1. For improved performance, configuring [Database Load Balancing](../postgresql/database_load_balancing.md) + with multiple read replicas is recommended. See [Configure GitLab using an external PostgreSQL service](../postgresql/external.md) for further configuration steps. ### Standalone PostgreSQL using Omnibus GitLab +The recommended Omnibus GitLab configuration for a PostgreSQL cluster with +replication and failover requires: + +- A minimum of three PostgreSQL nodes. +- A minimum of three Consul server nodes. +- A minimum of three PgBouncer nodes that track and handle primary database reads and writes. + - An [internal load balancer](#configure-the-internal-load-balancer) (TCP) to balance requests between the PgBouncer nodes. +- [Database Load Balancing](../postgresql/database_load_balancing.md) enabled. + + A local PgBouncer service to be configured on each PostgreSQL node. Note that this is separate from the main PgBouncer cluster that tracks the primary. + The following IPs will be used as an example: - `10.6.0.31`: PostgreSQL primary @@ -837,8 +853,8 @@ in the second step, do not supply the `EXTERNAL_URL` value. 1. On every database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: ```ruby - # Disable all components except Patroni and Consul - roles(['patroni_role']) + # Disable all components except Patroni, PgBouncer and Consul + roles(['patroni_role', 'pgbouncer_role']) # PostgreSQL configuration postgresql['listen_address'] = '0.0.0.0' @@ -883,6 +899,15 @@ in the second step, do not supply the `EXTERNAL_URL` value. # Replace 10.6.0.0/24 with Network Address postgresql['trust_auth_cidr_addresses'] = %w(10.6.0.0/24 127.0.0.1/32) + # Local PgBouncer service for Database Load Balancing + pgbouncer['databases'] = { + gitlabhq_production: { + host: "127.0.0.1", + user: "pgbouncer", + password: '<pgbouncer_password_hash>' + } + } + # Set the network addresses that the exporters will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' postgres_exporter['listen_address'] = '0.0.0.0:9187' @@ -943,9 +968,11 @@ If the 'State' column for any node doesn't say "running", check the </a> </div> -## Configure PgBouncer +### Configure PgBouncer + +Now that the PostgreSQL servers are all set up, let's configure PgBouncer +for tracking and handling reads/writes to the primary database. -Now that the PostgreSQL servers are all set up, let's configure PgBouncer. The following IPs will be used as an example: - `10.6.0.21`: PgBouncer 1 @@ -1604,8 +1631,8 @@ To configure the Sidekiq nodes, one each one: gitlab_rails['db_host'] = '10.6.0.40' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs + ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1764,6 +1791,8 @@ On each node perform the following: gitlab_rails['db_host'] = '10.6.0.20' # internal load balancer IP gitlab_rails['db_port'] = 6432 gitlab_rails['db_password'] = '<postgresql_user_password>' + gitlab_rails['db_load_balancing'] = { 'hosts' => ['10.6.0.31', '10.6.0.32', '10.6.0.33'] } # PostgreSQL IPs + # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -2153,25 +2182,21 @@ For all PaaS solutions that involve configuring instances, it is strongly recomm ```plantuml @startuml 5k +skinparam linetype ortho card "Kubernetes via Helm Charts" as kubernetes { card "**External Load Balancer**" as elb #6a9be7 together { - collections "**Webservice** x5" as gitlab #32CD32 - collections "**Sidekiq** x3" as sidekiq #ff8dd1 + collections "**Webservice** x4" as gitlab #32CD32 + collections "**Sidekiq** x4" as sidekiq #ff8dd1 } - card "**Prometheus + Grafana**" as monitor #7FFFD4 card "**Supporting Services**" as support } card "**Internal Load Balancer**" as ilb #9370DB - -card "**Consul + Sentinel**" as consul_sentinel { - collections "**Consul** x3" as consul #e76a9b - collections "**Redis Sentinel** x3" as sentinel #e6e727 -} +collections "**Consul** x3" as consul #e76a9b card "Gitaly Cluster" as gitaly_cluster { collections "**Praefect** x3" as praefect #FF8C00 @@ -2191,41 +2216,33 @@ card "Database" as database { postgres_primary .[#4EA7FF]> postgres_secondary } -card "Redis" as redis { +card "redis" as redis { collections "**Redis** x3" as redis_nodes #FF6347 - - redis_nodes <.[#FF6347]- sentinel } cloud "**Object Storage**" as object_storage #white elb -[#6a9be7]-> gitlab -elb -[#6a9be7]-> monitor +elb -[hidden]-> sidekiq elb -[hidden]-> support gitlab -[#32CD32]--> ilb -gitlab -[#32CD32]-> object_storage -gitlab -[#32CD32]---> redis -gitlab -[hidden]--> consul +gitlab -[#32CD32]r--> object_storage +gitlab -[#32CD32,norank]----> redis +gitlab -[#32CD32]----> database sidekiq -[#ff8dd1]--> ilb -sidekiq -[#ff8dd1]-> object_storage -sidekiq -[#ff8dd1]---> redis -sidekiq -[hidden]--> consul - -ilb -[#9370DB]-> gitaly_cluster -ilb -[#9370DB]-> database +sidekiq -[#ff8dd1]r--> object_storage +sidekiq -[#ff8dd1,norank]----> redis +sidekiq .[#ff8dd1]----> database -consul .[#e76a9b]-> database -consul .[#e76a9b]-> gitaly_cluster -consul .[#e76a9b,norank]--> redis +ilb -[#9370DB]--> gitaly_cluster +ilb -[#9370DB]--> database +ilb -[hidden,norank]--> redis -monitor .[#7FFFD4]> consul -monitor .[#7FFFD4]-> database -monitor .[#7FFFD4]-> gitaly_cluster -monitor .[#7FFFD4,norank]--> redis -monitor .[#7FFFD4]> ilb -monitor .[#7FFFD4,norank]u--> elb +consul .[#e76a9b]--> database +consul .[#e76a9b,norank]--> gitaly_cluster +consul .[#e76a9b]--> redis @enduml ``` diff --git a/doc/development/graphql_guide/authorization.md b/doc/development/graphql_guide/authorization.md index d7edd01cda2..717a6d29fbc 100644 --- a/doc/development/graphql_guide/authorization.md +++ b/doc/development/graphql_guide/authorization.md @@ -43,6 +43,11 @@ such as short pages, which can expose the presence of confidential resources. See [`authorization_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/features/authorization_spec.rb) for examples of all the authorization schemes discussed here. +<!-- + NOTE: if you change this heading (or the location to this file), make sure to update + the referenced link in rubocop/cop/graphql/authorize_types.rb +--> + ## Type authorization Authorize a type by passing an ability to the `authorize` method. All diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb new file mode 100644 index 00000000000..33a24e61a3f --- /dev/null +++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class BadgesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::RestExtractor, + query: BulkImports::Common::Rest::GetBadgesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + return if data.blank? + # Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project. + return if context.entity.project? && group_badge?(data) + + { + name: data['name'], + link_url: data['link_url'], + image_url: data['image_url'] + } + end + + def load(context, data) + return if data.blank? + + if context.entity.project? + context.portable.project_badges.create!(data) + else + context.portable.badges.create!(data) + end + end + + private + + def group_badge?(data) + data['kind'] == 'group' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb index 79ffdd9a1f6..60b2ebcc552 100644 --- a/lib/bulk_imports/groups/rest/get_badges_query.rb +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Rest module GetBadgesQuery extend self def to_h(context) + resource = context.entity.pluralized_name encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) { - resource: ['groups', encoded_full_path, 'badges'].join('/'), + resource: [resource, encoded_full_path, 'badges'].join('/'), query: { page: context.tracker.next_page } diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb deleted file mode 100644 index 8569ff3f77a..00000000000 --- a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class BadgesPipeline - include Pipeline - - extractor BulkImports::Common::Extractors::RestExtractor, - query: BulkImports::Groups::Rest::GetBadgesQuery - - transformer Common::Transformers::ProhibitedAttributesTransformer - - def transform(_, data) - return if data.blank? - - { - name: data['name'], - link_url: data['link_url'], - image_url: data['image_url'] - } - end - - def load(context, data) - return if data.blank? - - context.group.badges.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 241dd428dd5..6631c212913 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -32,7 +32,7 @@ module BulkImports stage: 1 }, badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, stage: 1 }, boards: { diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index cc5968194a4..d95130b59e0 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -23,6 +23,10 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 2 }, + badges: { + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 870cf25984b..68a0c15480a 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -32,12 +32,12 @@ module Gitlab # Models using single-type inheritance (STI) don't work with # reltuple count estimates. We just have to ignore them and # use another strategy to compute them. - def non_sti_models + def non_sti_models(models) models.reject { |model| sti_model?(model) } end - def non_sti_table_names - non_sti_models.map(&:table_name) + def non_sti_table_names(models) + non_sti_models(models).map(&:table_name) end def sti_model?(model) @@ -45,21 +45,34 @@ module Gitlab model.base_class != model end - def table_names - models.map(&:table_name) + def table_to_model_mapping + @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + end + + def table_to_model(table_name) + table_to_model_mapping[table_name] end def size_estimates(check_statistics: true) - table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } - - # Querying tuple stats only works on the primary. Due to load balancing, the - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases - get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| - model = table_to_model[row.table_name] - data[model] = row.estimate + results = {} + + models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db| + base_model = Gitlab::Database.database_base_models[db_name] + tables = non_sti_table_names(models_for_db) + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + base_model.transaction do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + get_statistics(tables, check_statistics: check_statistics).each do |row| + model = table_to_model(row.table_name) + results[model] = row.estimate + end + end end end + + results end # Generates the PostgreSQL query to return the tuples for tables diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 489bc0aacea..92c8de9aeac 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -61,7 +61,7 @@ module Gitlab #{where_clause(model)} SQL - rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases + rows = model.connection.select_all(query) Integer(rows.first['count']) end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5a553a6ef56..5695f2f1c14 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -261,6 +261,7 @@ issuable_severities: :gitlab_main issuable_slas: :gitlab_main issue_assignees: :gitlab_main issue_customer_relations_contacts: :gitlab_main +issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index bdaf0d35a83..9bc68e9d7c9 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -258,7 +258,9 @@ module Gitlab # We keep track of the estimated number of tuples to reason later # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end migration.save! migration diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb index 0ce9eebc14c..bd582d903c6 100644 --- a/lib/gitlab/database/pg_class.rb +++ b/lib/gitlab/database/pg_class.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PgClass < ActiveRecord::Base + class PgClass < SharedModel self.table_name = 'pg_class' def self.for_table(relname) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index a42455aab2a..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,11 +32,11 @@ module Gitlab def execute raise ProjectNotFound if project.nil? - create_issue! + create_issue_or_note if from_address add_email_participant - send_thank_you_email + send_thank_you_email unless reply_email? end end @@ -82,6 +82,14 @@ module Gitlab project.present? && slug == project.full_path_slug end + def create_issue_or_note + if reply_email? + create_note_from_reply_email + else + create_issue! + end + end + def create_issue! @issue = ::Issues::CreateService.new( project: project, @@ -97,11 +105,35 @@ module Gitlab raise InvalidIssueError unless @issue.persisted? + begin + ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e) + end + if service_desk_setting&.issue_template_missing? - create_template_not_found_note(@issue) + create_template_not_found_note + end + end + + def issue_from_reply_to + strong_memoize(:issue_from_reply_to) do + next unless mail.in_reply_to + + Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue end end + def reply_email? + issue_from_reply_to.present? + end + + def create_note_from_reply_email + @issue = issue_from_reply_to + + create_note(message_including_reply) + end + def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) @@ -124,7 +156,7 @@ module Gitlab end end - def create_template_not_found_note(issue) + def create_template_not_found_note issue_template_key = service_desk_setting&.issue_template_key warning_note = <<-MD.strip_heredoc @@ -132,15 +164,15 @@ module Gitlab Please check service desk settings and update the file to be used. MD - note_params = { - noteable: issue, - note: warning_note - } + create_note(warning_note) + end + def create_note(note) ::Notes::CreateService.new( project, User.support_bot, - note_params + noteable: @issue, + note: note ).execute end @@ -157,6 +189,8 @@ module Gitlab end def add_email_participant + return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + @issue.issue_email_participants.create(email: from_address) end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d815dd284ba..2172efba83c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -747,6 +747,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id + - :email_message_id merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index c917debd3d9..9bd2309d2b7 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true +require 'securerandom' require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/container_v1beta1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' +require 'google/apis/iam_v1' module GoogleApi module CloudPlatform @@ -83,6 +85,51 @@ module GoogleApi m[1] if m end + def list_projects + result = [] + + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + response = service.fetch_all(items: :projects) do |token| + service.list_projects + end + + # Google API results are paged by default, so we need to iterate through + response.each do |project| + result.append(project) + end + + result + end + + def create_service_account(gcp_project_id, display_name, description) + name = "projects/#{gcp_project_id}" + + # initialize google iam service + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + # generate account id + random_account_id = "gitlab-" + SecureRandom.hex(11) + + body_params = { account_id: random_account_id, + service_account: { display_name: display_name, + description: description } } + + request_body = Google::Apis::IamV1::CreateServiceAccountRequest.new(**body_params) + service.create_service_account(name, request_body) + end + + def create_service_account_key(gcp_project_id, service_account_id) + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + name = "projects/#{gcp_project_id}/serviceAccounts/#{service_account_id}" + request_body = Google::Apis::IamV1::CreateServiceAccountKeyRequest.new + service.create_service_account_key(name, request_body) + end + private def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 545ab7c7cad..dd1994954a7 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -16212,6 +16212,9 @@ msgstr "" msgid "Google Cloud Project" msgstr "" +msgid "Google Cloud authorizations required" +msgstr "" + msgid "Google authentication is not %{link_start}properly configured%{link_end}. Ask your GitLab administrator if you want to use this service." msgstr "" @@ -31628,6 +31631,9 @@ msgstr "" msgid "Service URL" msgstr "" +msgid "Service account generated successfully" +msgstr "" + msgid "Service ping is disabled in your configuration file, and cannot be enabled through this form." msgstr "" diff --git a/rubocop/cop/graphql/authorize_types.rb b/rubocop/cop/graphql/authorize_types.rb index d5866aa0aaf..c96919343d6 100644 --- a/rubocop/cop/graphql/authorize_types.rb +++ b/rubocop/cop/graphql/authorize_types.rb @@ -5,7 +5,7 @@ module RuboCop module Graphql class AuthorizeTypes < RuboCop::Cop::Cop MSG = 'Add an `authorize :ability` call to the type: '\ - 'https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#type-authorization' + 'https://docs.gitlab.com/ee/development/graphql_guide/authorization.html#type-authorization' # We want to exclude our own basetypes and scalars ALLOWED_TYPES = %w[BaseEnum BaseEdge BaseScalar BasePermissionType MutationType SubscriptionType diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index 0d76c0d33aa..88913b4e972 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -147,6 +147,13 @@ function disable_sign_ups() { fi } +function create_sample_projects() { + local create_sample_projects_rb="root_user = User.find_by_username('root'); 1.times { |i| params = { namespace_id: root_user.namespace.id, name: 'sample-project' + i.to_s, path: 'sample-project' + i.to_s, template_name: 'sample' }; ::Projects::CreateFromTemplateService.new(root_user, params).execute }" + + # Queue jobs to create sample projects for root user namespace from sample data project template + retry "run_task \"${create_sample_projects_rb}\"" +} + function check_kube_domain() { echoinfo "Checking that Kube domain exists..." true diff --git a/spec/controllers/google_api/authorizations_controller_spec.rb b/spec/controllers/google_api/authorizations_controller_spec.rb index 3dd2cc307d5..3bf50f98791 100644 --- a/spec/controllers/google_api/authorizations_controller_spec.rb +++ b/spec/controllers/google_api/authorizations_controller_spec.rb @@ -88,5 +88,26 @@ RSpec.describe GoogleApi::AuthorizationsController do it_behaves_like 'access denied' end + + context 'user logs in but declines authorizations' do + subject { get :callback, params: { error: 'xxx', state: state } } + + let(:session_key) { 'session-key' } + let(:redirect_uri) { 'example.com' } + let(:error_uri) { 'error.com' } + let(:state) { session_key } + + before do + session[GoogleApi::CloudPlatform::Client.session_key_for_redirect_uri(session_key)] = redirect_uri + session[:error_uri] = error_uri + allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |instance| + allow(instance).to receive(:get_token).and_return([token, expires_at]) + end + end + + it 'redirects to error uri' do + expect(subject).to redirect_to(error_uri) + end + end end end diff --git a/spec/controllers/projects/raw_controller_spec.rb b/spec/controllers/projects/raw_controller_spec.rb index a81173ccaac..79da18f2d6d 100644 --- a/spec/controllers/projects/raw_controller_spec.rb +++ b/spec/controllers/projects/raw_controller_spec.rb @@ -101,7 +101,7 @@ RSpec.describe Projects::RawController do expect(response).to have_gitlab_http_status(:too_many_requests) end - it 'logs the event on auth.log' do + it 'logs the event on auth.log', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/345889' do attributes = { message: 'Application_Rate_Limiter_Request', env: :raw_blob_request_limit, diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 521b4cd4002..9be09179c6c 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -53,6 +53,7 @@ RSpec.describe 'Database schema' do identities: %w[user_id], import_failures: %w[project_id], issues: %w[last_edited_by_id state_id], + issue_emails: %w[email_message_id], jira_tracker_data: %w[jira_issue_transition_id], keys: %w[user_id], label_links: %w[target_id], diff --git a/spec/factories/issue_emails.rb b/spec/factories/issue_emails.rb new file mode 100644 index 00000000000..edf07aab0cd --- /dev/null +++ b/spec/factories/issue_emails.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :issue_email, class: 'Issue::Email' do + issue + email_message_id { generate(:short_text) } + end +end diff --git a/spec/factories/sequences.rb b/spec/factories/sequences.rb index 0edc2b6027d..56f1fa162bf 100644 --- a/spec/factories/sequences.rb +++ b/spec/factories/sequences.rb @@ -21,4 +21,5 @@ FactoryBot.define do sequence(:jira_branch) { |n| "feature/PROJ-#{n}" } sequence(:job_name) { |n| "job #{n}" } sequence(:work_item_type_name) { |n| "bug#{n}" } + sequence(:short_text) { |n| "someText#{n}" } end diff --git a/spec/features/groups/import_export/import_file_spec.rb b/spec/features/groups/import_export/import_file_spec.rb index 76d17c4409d..3d23451feef 100644 --- a/spec/features/groups/import_export/import_file_spec.rb +++ b/spec/features/groups/import_export/import_file_spec.rb @@ -85,7 +85,7 @@ RSpec.describe 'Import/Export - Group Import', :js do context 'when the user uploads an invalid export file' do let(:file) { File.join(Rails.root, 'spec', %w[fixtures big-image.png]) } - it 'displays an error' do + it 'displays an error', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/343995' do visit new_group_path click_link 'Import group' diff --git a/spec/fixtures/emails/service_desk_forwarded.eml b/spec/fixtures/emails/service_desk_forwarded.eml index ab509cf55af..45ac419e42f 100644 --- a/spec/fixtures/emails/service_desk_forwarded.eml +++ b/spec/fixtures/emails/service_desk_forwarded.eml @@ -8,7 +8,7 @@ Date: Thu, 13 Jun 2013 17:03:48 -0400 From: Jake the Dog <jake.g@adventuretime.ooo> To: support@adventuretime.ooo Delivered-To: support@adventuretime.ooo -Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=fdskbsf@mail.gmail.com> Subject: The message subject! @all Mime-Version: 1.0 Content-Type: text/plain; diff --git a/spec/fixtures/emails/service_desk_reply.eml b/spec/fixtures/emails/service_desk_reply.eml new file mode 100644 index 00000000000..8e1d9aaf2d3 --- /dev/null +++ b/spec/fixtures/emails/service_desk_reply.eml @@ -0,0 +1,23 @@ +Return-Path: <alan@adventuretime.ooo> +Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400 +Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400 +Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+email-test-project_id-issue-@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700 +Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700 +Date: Thu, 13 Jun 2013 17:03:48 -0400 +From: Jake the Dog <alan@adventuretime.ooo> +To: incoming+email-test-project_id-issue-@appmail.adventuretime.ooo +Message-ID: <CAH_Wr+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +In-Reply-To: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Subject: The message subject! @all +Mime-Version: 1.0 +Content-Type: text/plain; + charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +X-Sieve: CMU Sieve 2.2 +X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, + 13 Jun 2013 14:03:48 -0700 (PDT) +X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1 + +Service desk reply! + +/label ~label2 diff --git a/spec/frontend/notes/components/__snapshots__/notes_app_spec.js.snap b/spec/frontend/notes/components/__snapshots__/notes_app_spec.js.snap new file mode 100644 index 00000000000..5f4b3e04a79 --- /dev/null +++ b/spec/frontend/notes/components/__snapshots__/notes_app_spec.js.snap @@ -0,0 +1,17 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`note_app when sort direction is asc shows skeleton notes after the loaded discussions 1`] = ` +"<ul id=\\"notes-list\\" class=\\"notes main-notes-list timeline\\"> + <noteable-discussion-stub discussion=\\"[object Object]\\" renderdifffile=\\"true\\" helppagepath=\\"\\" isoverviewtab=\\"true\\"></noteable-discussion-stub> + <skeleton-loading-container-stub></skeleton-loading-container-stub> + <discussion-filter-note-stub style=\\"display: none;\\"></discussion-filter-note-stub> +</ul>" +`; + +exports[`note_app when sort direction is desc shows skeleton notes before the loaded discussions 1`] = ` +"<ul id=\\"notes-list\\" class=\\"notes main-notes-list timeline\\"> + <skeleton-loading-container-stub></skeleton-loading-container-stub> + <noteable-discussion-stub discussion=\\"[object Object]\\" renderdifffile=\\"true\\" helppagepath=\\"\\" isoverviewtab=\\"true\\"></noteable-discussion-stub> + <discussion-filter-note-stub style=\\"display: none;\\"></discussion-filter-note-stub> +</ul>" +`; diff --git a/spec/frontend/notes/components/notes_app_spec.js b/spec/frontend/notes/components/notes_app_spec.js index e91767687e8..84d94857fe5 100644 --- a/spec/frontend/notes/components/notes_app_spec.js +++ b/spec/frontend/notes/components/notes_app_spec.js @@ -374,6 +374,9 @@ describe('note_app', () => { beforeEach(() => { store = createStore(); store.state.discussionSortOrder = constants.DESC; + store.state.isLoading = true; + store.state.discussions = [mockData.discussionMock]; + wrapper = shallowMount(NotesApp, { propsData, store, @@ -386,11 +389,18 @@ describe('note_app', () => { it('finds CommentForm before notes list', () => { expect(getComponentOrder()).toStrictEqual([TYPE_COMMENT_FORM, TYPE_NOTES_LIST]); }); + + it('shows skeleton notes before the loaded discussions', () => { + expect(wrapper.find('#notes-list').html()).toMatchSnapshot(); + }); }); describe('when sort direction is asc', () => { beforeEach(() => { store = createStore(); + store.state.isLoading = true; + store.state.discussions = [mockData.discussionMock]; + wrapper = shallowMount(NotesApp, { propsData, store, @@ -403,6 +413,10 @@ describe('note_app', () => { it('finds CommentForm after notes list', () => { expect(getComponentOrder()).toStrictEqual([TYPE_NOTES_LIST, TYPE_COMMENT_FORM]); }); + + it('shows skeleton notes after the loaded discussions', () => { + expect(wrapper.find('#notes-list').html()).toMatchSnapshot(); + }); }); describe('when multiple draft types are present', () => { diff --git a/spec/helpers/tab_helper_spec.rb b/spec/helpers/tab_helper_spec.rb index e5e88466946..eaf2a4d11c1 100644 --- a/spec/helpers/tab_helper_spec.rb +++ b/spec/helpers/tab_helper_spec.rb @@ -161,18 +161,24 @@ RSpec.describe TabHelper do describe 'gl_tab_counter_badge' do it 'creates a tab counter badge' do - expect(gl_tab_counter_badge(1)).to eq('<span class="badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge">1</span>') + expect(helper.gl_tab_counter_badge(1)).to eq( + '<span class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge">1</span>' + ) end context 'with extra classes' do it 'creates a tab counter badge with the correct class attribute' do - expect(gl_tab_counter_badge(1, { class: 'js-test' })).to eq('<span class="js-test badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge">1</span>') + expect(helper.gl_tab_counter_badge(1, { class: 'js-test' })).to eq( + '<span class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge js-test">1</span>' + ) end end context 'with data attributes' do it 'creates a tab counter badge with the data attributes' do - expect(gl_tab_counter_badge(1, { data: { some_attribute: 'foo' } })).to eq('<span class="badge badge-muted badge-pill gl-badge sm gl-tab-counter-badge" data-some-attribute="foo">1</span>') + expect(helper.gl_tab_counter_badge(1, { data: { some_attribute: 'foo' } })).to eq( + '<span data-some-attribute="foo" class="gl-badge badge badge-pill badge-muted sm gl-tab-counter-badge">1</span>' + ) end end end diff --git a/spec/lib/bulk_imports/common/pipelines/badges_pipeline_spec.rb b/spec/lib/bulk_imports/common/pipelines/badges_pipeline_spec.rb new file mode 100644 index 00000000000..6c5465c8a66 --- /dev/null +++ b/spec/lib/bulk_imports/common/pipelines/badges_pipeline_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Common::Pipelines::BadgesPipeline do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project) } + + let(:entity) { create(:bulk_import_entity, group: group) } + let(:tracker) { create(:bulk_import_tracker, entity: entity) } + let(:context) { BulkImports::Pipeline::Context.new(tracker) } + + subject(:pipeline) { described_class.new(context) } + + describe '#run' do + let(:first_page) { extracted_data(has_next_page: true) } + let(:last_page) { extracted_data(name: 'badge2') } + + before do + allow_next_instance_of(BulkImports::Common::Extractors::RestExtractor) do |extractor| + allow(extractor).to receive(:extract).and_return(first_page, last_page) + end + end + + it 'imports a group badge' do + expect { pipeline.run }.to change(Badge, :count).by(2) + + badge = group.badges.last + + expect(badge.name).to eq('badge2') + expect(badge.link_url).to eq(badge_data['link_url']) + expect(badge.image_url).to eq(badge_data['image_url']) + end + + context 'when project entity' do + let(:first_page) { extracted_data(has_next_page: true) } + let(:last_page) { extracted_data(name: 'badge2', kind: 'project') } + let(:entity) { create(:bulk_import_entity, :project_entity, project: project) } + + it 'imports a project badge & skips group badge' do + expect { pipeline.run }.to change(Badge, :count).by(1) + + badge = project.badges.last + + expect(badge.name).to eq('badge2') + expect(badge.link_url).to eq(badge_data['link_url']) + expect(badge.image_url).to eq(badge_data['image_url']) + expect(badge.type).to eq('ProjectBadge') + end + end + + describe '#transform' do + it 'return transformed badge hash' do + badge = subject.transform(context, badge_data) + + expect(badge[:name]).to eq('badge') + expect(badge[:link_url]).to eq(badge_data['link_url']) + expect(badge[:image_url]).to eq(badge_data['image_url']) + expect(badge.keys).to contain_exactly(:name, :link_url, :image_url) + end + + context 'when data is blank' do + it 'does nothing when the data is blank' do + expect(subject.transform(context, nil)).to be_nil + end + end + + context 'when project entity & group badge' do + let(:entity) { create(:bulk_import_entity, :project_entity, project: project) } + + it 'returns' do + expect(subject.transform(context, { 'name' => 'test', 'kind' => 'group' })).to be_nil + end + end + end + + def badge_data(name = 'badge', kind = 'group') + { + 'name' => name, + 'link_url' => 'https://gitlab.example.com', + 'image_url' => 'https://gitlab.example.com/image.png', + 'kind' => kind + } + end + + def extracted_data(name: 'badge', kind: 'group', has_next_page: false) + page_info = { + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? '2' : nil + } + + BulkImports::Pipeline::ExtractedData.new(data: [badge_data(name, kind)], page_info: page_info) + end + end +end diff --git a/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb b/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb new file mode 100644 index 00000000000..0a04c0a2243 --- /dev/null +++ b/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Common::Rest::GetBadgesQuery do + describe '.to_h' do + shared_examples 'resource and page info query' do + let(:tracker) { create(:bulk_import_tracker, entity: entity) } + let(:context) { BulkImports::Pipeline::Context.new(tracker) } + let(:encoded_full_path) { ERB::Util.url_encode(entity.source_full_path) } + + it 'returns correct query and page info' do + expected = { + resource: [entity.pluralized_name, encoded_full_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } + } + + expect(described_class.to_h(context)).to eq(expected) + end + end + + context 'when entity is group' do + let(:entity) { create(:bulk_import_entity) } + + include_examples 'resource and page info query' + end + + context 'when entity is project' do + let(:entity) { create(:bulk_import_entity, :project_entity) } + + include_examples 'resource and page info query' + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb deleted file mode 100644 index 9fa35c4707d..00000000000 --- a/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Groups::Pipelines::BadgesPipeline do - let_it_be(:user) { create(:user) } - let_it_be(:group) { create(:group) } - - let_it_be(:entity) do - create( - :bulk_import_entity, - source_full_path: 'source/full/path', - destination_name: 'My Destination Group', - destination_namespace: group.full_path, - group: group - ) - end - - let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } - - subject { described_class.new(context) } - - describe '#run' do - it 'imports a group badge' do - first_page = extracted_data(has_next_page: true) - last_page = extracted_data(name: 'badge2') - - allow_next_instance_of(BulkImports::Common::Extractors::RestExtractor) do |extractor| - allow(extractor) - .to receive(:extract) - .and_return(first_page, last_page) - end - - expect { subject.run }.to change(Badge, :count).by(2) - - badge = group.badges.last - - expect(badge.name).to eq('badge2') - expect(badge.link_url).to eq(badge_data['link_url']) - expect(badge.image_url).to eq(badge_data['image_url']) - end - - describe '#load' do - it 'creates a badge' do - expect { subject.load(context, badge_data) }.to change(Badge, :count).by(1) - - badge = group.badges.first - - badge_data.each do |key, value| - expect(badge[key]).to eq(value) - end - end - - it 'does nothing when the data is blank' do - expect { subject.load(context, nil) }.not_to change(Badge, :count) - end - end - - describe '#transform' do - it 'return transformed badge hash' do - badge = subject.transform(context, badge_data) - - expect(badge[:name]).to eq('badge') - expect(badge[:link_url]).to eq(badge_data['link_url']) - expect(badge[:image_url]).to eq(badge_data['image_url']) - expect(badge.keys).to contain_exactly(:name, :link_url, :image_url) - end - - context 'when data is blank' do - it 'does nothing when the data is blank' do - expect(subject.transform(context, nil)).to be_nil - end - end - end - - describe 'pipeline parts' do - it { expect(described_class).to include_module(BulkImports::Pipeline) } - it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } - - it 'has extractors' do - expect(described_class.get_extractor) - .to eq( - klass: BulkImports::Common::Extractors::RestExtractor, - options: { - query: BulkImports::Groups::Rest::GetBadgesQuery - } - ) - end - - it 'has transformers' do - expect(described_class.transformers) - .to contain_exactly( - { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } - ) - end - end - - def badge_data(name = 'badge') - { - 'name' => name, - 'link_url' => 'https://gitlab.example.com', - 'image_url' => 'https://gitlab.example.com/image.png' - } - end - - def extracted_data(name: 'badge', has_next_page: false) - page_info = { - 'has_next_page' => has_next_page, - 'next_page' => has_next_page ? '2' : nil - } - - BulkImports::Pipeline::ExtractedData.new(data: [badge_data(name)], page_info: page_info) - end - end -end diff --git a/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb b/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb deleted file mode 100644 index eef6848e118..00000000000 --- a/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Groups::Rest::GetBadgesQuery do - describe '.to_h' do - it 'returns query resource and page info' do - entity = create(:bulk_import_entity) - tracker = create(:bulk_import_tracker, entity: entity) - context = BulkImports::Pipeline::Context.new(tracker) - encoded_full_path = ERB::Util.url_encode(entity.source_full_path) - expected = { - resource: ['groups', encoded_full_path, 'badges'].join('/'), - query: { - page: context.tracker.next_page - } - } - - expect(described_class.to_h(context)).to eq(expected) - end - end -end diff --git a/spec/lib/bulk_imports/groups/stage_spec.rb b/spec/lib/bulk_imports/groups/stage_spec.rb index 5719acac4d7..a7acd661282 100644 --- a/spec/lib/bulk_imports/groups/stage_spec.rb +++ b/spec/lib/bulk_imports/groups/stage_spec.rb @@ -13,7 +13,7 @@ RSpec.describe BulkImports::Groups::Stage do [1, BulkImports::Groups::Pipelines::MembersPipeline], [1, BulkImports::Common::Pipelines::LabelsPipeline], [1, BulkImports::Common::Pipelines::MilestonesPipeline], - [1, BulkImports::Groups::Pipelines::BadgesPipeline], + [1, BulkImports::Common::Pipelines::BadgesPipeline], [2, BulkImports::Common::Pipelines::BoardsPipeline] ] end diff --git a/spec/lib/bulk_imports/projects/stage_spec.rb b/spec/lib/bulk_imports/projects/stage_spec.rb index cbc0ea667b7..ee197cb1c29 100644 --- a/spec/lib/bulk_imports/projects/stage_spec.rb +++ b/spec/lib/bulk_imports/projects/stage_spec.rb @@ -11,6 +11,7 @@ RSpec.describe BulkImports::Projects::Stage do [1, BulkImports::Projects::Pipelines::RepositoryPipeline], [2, BulkImports::Common::Pipelines::LabelsPipeline], [2, BulkImports::Common::Pipelines::MilestonesPipeline], + [2, BulkImports::Common::Pipelines::BadgesPipeline], [3, BulkImports::Projects::Pipelines::IssuesPipeline], [3, BulkImports::Projects::Pipelines::SnippetsPipeline], [4, BulkImports::Common::Pipelines::BoardsPipeline], diff --git a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb index 9d49db1f018..e7b9c5fcd02 100644 --- a/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb @@ -5,24 +5,24 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do before do create_list(:project, 3) - create(:identity) + create_list(:ci_instance_variable, 2) end subject { described_class.new(models).count } describe '#count' do - let(:models) { [Project, Identity] } + let(:models) { [Project, Ci::InstanceVariable] } context 'when reltuples is up to date' do before do - ActiveRecord::Base.connection.execute('ANALYZE projects') - ActiveRecord::Base.connection.execute('ANALYZE identities') + Project.connection.execute('ANALYZE projects') + Ci::InstanceVariable.connection.execute('ANALYZE ci_instance_variables') end it 'uses statistics to do the count' do models.each { |model| expect(model).not_to receive(:count) } - expect(subject).to eq({ Project => 3, Identity => 1 }) + expect(subject).to eq({ Project => 3, Ci::InstanceVariable => 2 }) end end @@ -31,7 +31,7 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do before do models.each do |model| - ActiveRecord::Base.connection.execute("ANALYZE #{model.table_name}") + model.connection.execute("ANALYZE #{model.table_name}") end end @@ -45,7 +45,9 @@ RSpec.describe Gitlab::Database::Count::ReltuplesCountStrategy do context 'insufficient permissions' do it 'returns an empty hash' do - allow(ActiveRecord::Base).to receive(:transaction).and_raise(PG::InsufficientPrivilege) + Gitlab::Database.database_base_models.each_value do |base_model| + allow(base_model).to receive(:transaction).and_raise(PG::InsufficientPrivilege) + end expect(subject).to eq({}) end diff --git a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb index 2f261aebf02..37d3e13a7ab 100644 --- a/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb +++ b/spec/lib/gitlab/database/count/tablesample_count_strategy_spec.rb @@ -5,11 +5,12 @@ require 'spec_helper' RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do before do create_list(:project, 3) + create_list(:ci_instance_variable, 2) create(:identity) create(:group) end - let(:models) { [Project, Identity, Group, Namespace] } + let(:models) { [Project, Ci::InstanceVariable, Identity, Group, Namespace] } let(:strategy) { described_class.new(models) } subject { strategy.count } @@ -20,7 +21,8 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do Project => threshold + 1, Identity => threshold - 1, Group => threshold + 1, - Namespace => threshold + 1 + Namespace => threshold + 1, + Ci::InstanceVariable => threshold + 1 } end @@ -43,12 +45,14 @@ RSpec.describe Gitlab::Database::Count::TablesampleCountStrategy do expect(Project).not_to receive(:count) expect(Group).not_to receive(:count) expect(Namespace).not_to receive(:count) + expect(Ci::InstanceVariable).not_to receive(:count) result = subject expect(result[Project]).to eq(3) expect(result[Group]).to eq(1) # 1-Group, 3 namespaces for each project and 3 project namespaces for each project expect(result[Namespace]).to eq(7) + expect(result[Ci::InstanceVariable]).to eq(2) end end diff --git a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb index c579027788d..7c34fb1a926 100644 --- a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb @@ -12,6 +12,8 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do let(:email_raw) { email_fixture('emails/service_desk.eml') } let(:author_email) { 'jake@adventuretime.ooo' } + let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com' } + let_it_be(:group) { create(:group, :private, name: "email") } let(:expected_description) do @@ -40,6 +42,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do expect(new_issue.all_references.all).to be_empty expect(new_issue.title).to eq("The message subject! @all") expect(new_issue.description).to eq(expected_description.strip) + expect(new_issue.email&.email_message_id).to eq(message_id) end it 'creates an issue_email_participant' do @@ -72,6 +75,95 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do it_behaves_like 'a new issue request' end + context 'when replying to issue creation email' do + def receive_reply + reply_email_raw = email_fixture('emails/service_desk_reply.eml') + + second_receiver = Gitlab::Email::Receiver.new(reply_email_raw) + second_receiver.execute + end + + context 'when an issue with message_id has been found' do + before do + receiver.execute + end + + subject do + receive_reply + end + + it 'does not create an additional issue' do + expect { subject }.not_to change { Issue.count } + end + + it 'adds a comment to the created issue' do + subject + + notes = Issue.last.notes + new_note = notes.first + + expect(notes.count).to eq(1) + expect(new_note.note).to eq("Service desk reply!\n\n`/label ~label2`") + expect(new_note.author).to eql(User.support_bot) + end + + it 'does not send thank you email' do + expect(Notify).not_to receive(:service_desk_thank_you_email) + + subject + end + + context 'when issue_email_participants FF is enabled' do + it 'creates 2 issue_email_participants' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(alan@adventuretime.ooo jake@adventuretime.ooo)) + end + end + + context 'when issue_email_participants FF is disabled' do + before do + stub_feature_flags(issue_email_participants: false) + end + + it 'creates only 1 issue_email_participant' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(jake@adventuretime.ooo)) + end + end + end + + context 'when an issue with message_id has not been found' do + subject do + receive_reply + end + + it 'creates a new issue correctly' do + expect { subject }.to change { Issue.count }.by(1) + + issue = Issue.last + + expect(issue.description).to eq("Service desk reply!\n\n`/label ~label2`") + end + + it 'sends thank you email once' do + expect(Notify).to receive(:service_desk_thank_you_email).once.and_return(double(deliver_later: true)) + + subject + end + + it 'creates 1 issue_email_participant' do + subject + + expect(Issue.last.issue_email_participants.map(&:email)) + .to match_array(%w(alan@adventuretime.ooo)) + end + end + end + context 'when using issue templates' do let_it_be(:user) { create(:user) } @@ -270,6 +362,20 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do end end + context 'when issue email creation fails' do + before do + allow(::Issue::Email).to receive(:create!).and_raise(StandardError) + end + + it 'still creates a new issue' do + expect { receiver.execute }.to change { Issue.count }.by(1) + end + + it 'does not create issue email record' do + expect { receiver.execute }.not_to change { Issue::Email.count } + end + end + context 'when rate limiting is in effect', :freeze_time, :clean_gitlab_redis_rate_limiting do let(:receiver) { Gitlab::Email::Receiver.new(email_raw) } @@ -291,19 +397,19 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do rescue RateLimitedService::RateLimitedError end.to change { Issue.count }.by(1) end + end - context 'when requests are sent by different users' do - let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') } - let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) } + context 'when requests are sent by different users' do + let(:email_raw_2) { email_fixture('emails/service_desk_forwarded.eml') } + let(:receiver2) { Gitlab::Email::Receiver.new(email_raw_2) } - subject do - receiver.execute - receiver2.execute - end + subject do + receiver.execute + receiver2.execute + end - it 'creates 2 issues' do - expect { subject }.to change { Issue.count }.by(2) - end + it 'creates 2 issues' do + expect { subject }.to change { Issue.count }.by(2) end end @@ -389,6 +495,7 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do context 'when the email is forwarded through an alias' do let(:author_email) { 'jake.g@adventuretime.ooo' } let(:email_raw) { email_fixture('emails/service_desk_forwarded.eml') } + let(:message_id) { 'CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=fdskbsf@mail.gmail.com' } it_behaves_like 'a new issue request' end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index b474f5825fd..49052623436 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -61,6 +61,7 @@ issues: - pending_escalations - customer_relations_contacts - issue_customer_relations_contacts +- email work_item_type: - issues events: diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 9daa3b32fd1..2f7db15ae29 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -33,6 +33,7 @@ Issue: - health_status - external_key - issue_type +- email_message_id Event: - id - target_type diff --git a/spec/lib/google_api/cloud_platform/client_spec.rb b/spec/lib/google_api/cloud_platform/client_spec.rb index 3dd8f7c413e..3284c9cd0d1 100644 --- a/spec/lib/google_api/cloud_platform/client_spec.rb +++ b/spec/lib/google_api/cloud_platform/client_spec.rb @@ -209,4 +209,47 @@ RSpec.describe GoogleApi::CloudPlatform::Client do expect(subject.header).to eq({ 'User-Agent': 'GitLab/10.3 (GPN:GitLab;)' }) end end + + describe '#list_projects' do + subject { client.list_projects } + + let(:list_of_projects) { [{}, {}, {}] } + let(:next_page_token) { nil } + let(:operation) { double('projects': list_of_projects, 'next_page_token': next_page_token) } + + it 'calls Google Api CloudResourceManagerService#list_projects' do + expect_any_instance_of(Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService) + .to receive(:list_projects) + .and_return(operation) + is_expected.to eq(list_of_projects) + end + end + + describe '#create_service_account' do + subject { client.create_service_account(spy, spy, spy) } + + let(:operation) { double('Service Account') } + + it 'calls Google Api IamService#create_service_account' do + expect_any_instance_of(Google::Apis::IamV1::IamService) + .to receive(:create_service_account) + .with(any_args) + .and_return(operation) + is_expected.to eq(operation) + end + end + + describe '#create_service_account_key' do + subject { client.create_service_account_key(spy, spy) } + + let(:operation) { double('Service Account Key') } + + it 'class Google Api IamService#create_service_account_key' do + expect_any_instance_of(Google::Apis::IamV1::IamService) + .to receive(:create_service_account_key) + .with(any_args) + .and_return(operation) + is_expected.to eq(operation) + end + end end diff --git a/spec/models/bulk_imports/entity_spec.rb b/spec/models/bulk_imports/entity_spec.rb index cc66572cd6f..1be564904c7 100644 --- a/spec/models/bulk_imports/entity_spec.rb +++ b/spec/models/bulk_imports/entity_spec.rb @@ -252,4 +252,34 @@ RSpec.describe BulkImports::Entity, type: :model do .to eq("/groups/#{entity.encoded_source_full_path}/export_relations/download?relation=test") end end + + describe '#entity_type' do + it 'returns entity type' do + group_entity = build(:bulk_import_entity) + project_entity = build(:bulk_import_entity, :project_entity) + + expect(group_entity.entity_type).to eq('group') + expect(project_entity.entity_type).to eq('project') + end + end + + describe '#project?' do + it 'returns true if project entity' do + group_entity = build(:bulk_import_entity) + project_entity = build(:bulk_import_entity, :project_entity) + + expect(group_entity.project?).to eq(false) + expect(project_entity.project?).to eq(true) + end + end + + describe '#group?' do + it 'returns true if group entity' do + group_entity = build(:bulk_import_entity) + project_entity = build(:bulk_import_entity, :project_entity) + + expect(group_entity.group?).to eq(true) + expect(project_entity.group?).to eq(false) + end + end end diff --git a/spec/models/issue/email_spec.rb b/spec/models/issue/email_spec.rb new file mode 100644 index 00000000000..57cc7c7df66 --- /dev/null +++ b/spec/models/issue/email_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Issue::Email do + describe 'Associations' do + it { is_expected.to belong_to(:issue) } + end + + describe 'Validations' do + subject { build(:issue_email) } + + it { is_expected.to validate_presence_of(:issue) } + it { is_expected.to validate_uniqueness_of(:issue) } + it { is_expected.to validate_uniqueness_of(:email_message_id) } + it { is_expected.to validate_length_of(:email_message_id).is_at_most(1000) } + it { is_expected.to validate_presence_of(:email_message_id) } + end +end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index ba4429451d1..c63f2d79ad9 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -32,6 +32,7 @@ RSpec.describe Issue do it { is_expected.to have_and_belong_to_many(:self_managed_prometheus_alert_events) } it { is_expected.to have_many(:prometheus_alerts) } it { is_expected.to have_many(:issue_email_participants) } + it { is_expected.to have_one(:email) } it { is_expected.to have_many(:timelogs).autosave(true) } it { is_expected.to have_one(:incident_management_issuable_escalation_status) } it { is_expected.to have_many(:issue_customer_relations_contacts) } diff --git a/spec/requests/api/graphql/ci/jobs_spec.rb b/spec/requests/api/graphql/ci/jobs_spec.rb index f268036a78a..f0b886e5e50 100644 --- a/spec/requests/api/graphql/ci/jobs_spec.rb +++ b/spec/requests/api/graphql/ci/jobs_spec.rb @@ -90,7 +90,7 @@ RSpec.describe 'Query.project.pipeline' do create(:ci_build_need, build: deploy_job, name: 'rspec 1 2') end - it 'reports the build needs and previous stages with no duplicates' do + it 'reports the build needs and previous stages with no duplicates', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/346433' do post_graphql(query, current_user: user) expect(jobs_graphql_data).to contain_exactly( diff --git a/spec/requests/projects/google_cloud/service_accounts_controller_spec.rb b/spec/requests/projects/google_cloud/service_accounts_controller_spec.rb new file mode 100644 index 00000000000..6b4d1c490e2 --- /dev/null +++ b/spec/requests/projects/google_cloud/service_accounts_controller_spec.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# Mock Types +MockGoogleOAuth2Credentials = Struct.new(:app_id, :app_secret) +MockServiceAccount = Struct.new(:project_id, :unique_id) + +RSpec.describe Projects::GoogleCloud::ServiceAccountsController do + let_it_be(:project) { create(:project, :public) } + + describe 'GET index' do + let_it_be(:url) { "#{project_google_cloud_service_accounts_path(project)}" } + + let(:user_guest) { create(:user) } + let(:user_developer) { create(:user) } + let(:user_maintainer) { create(:user) } + let(:user_creator) { project.creator } + + let(:unauthorized_members) { [user_guest, user_developer] } + let(:authorized_members) { [user_maintainer, user_creator] } + + before do + project.add_guest(user_guest) + project.add_developer(user_developer) + project.add_maintainer(user_maintainer) + end + + context 'when a public request is made' do + it 'returns not found on GET request' do + get url + + expect(response).to have_gitlab_http_status(:not_found) + end + + it 'returns not found on POST request' do + post url + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when unauthorized members make requests' do + it 'returns not found on GET request' do + unauthorized_members.each do |unauthorized_member| + sign_in(unauthorized_member) + + get url + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + it 'returns not found on POST request' do + unauthorized_members.each do |unauthorized_member| + sign_in(unauthorized_member) + + post url + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + + context 'when authorized members make requests' do + it 'redirects on GET request' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + get url + + expect(response).to redirect_to(assigns(:authorize_url)) + end + end + + it 'redirects on POST request' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + post url + + expect(response).to redirect_to(assigns(:authorize_url)) + end + end + + context 'and user has successfully completed the google oauth2 flow' do + before do + allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client| + allow(client).to receive(:validate_token).and_return(true) + allow(client).to receive(:list_projects).and_return([{}, {}, {}]) + allow(client).to receive(:create_service_account).and_return(MockServiceAccount.new(123, 456)) + allow(client).to receive(:create_service_account_key).and_return({}) + end + end + + it 'returns success on GET' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + get url + + expect(response).to have_gitlab_http_status(:ok) + end + end + + it 'returns success on POST' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + post url, params: { gcp_project: 'prj1', environment: 'env1' } + + expect(response).to redirect_to(project_google_cloud_index_path(project)) + end + end + end + + context 'but google returns client error' do + before do + allow_next_instance_of(GoogleApi::CloudPlatform::Client) do |client| + allow(client).to receive(:validate_token).and_return(true) + allow(client).to receive(:list_projects).and_raise(Google::Apis::ClientError.new('')) + allow(client).to receive(:create_service_account).and_raise(Google::Apis::ClientError.new('')) + allow(client).to receive(:create_service_account_key).and_raise(Google::Apis::ClientError.new('')) + end + end + + it 'renders gcp_error template on GET' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + get url + + expect(response).to render_template(:gcp_error) + end + end + + it 'renders gcp_error template on POST' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + post url, params: { gcp_project: 'prj1', environment: 'env1' } + + expect(response).to render_template(:gcp_error) + end + end + end + + context 'but gitlab instance is not configured for google oauth2' do + before do + unconfigured_google_oauth2 = MockGoogleOAuth2Credentials.new('', '') + allow(Gitlab::Auth::OAuth::Provider).to receive(:config_for) + .with('google_oauth2') + .and_return(unconfigured_google_oauth2) + end + + it 'returns forbidden' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + get url + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + end + + context 'but feature flag is disabled' do + before do + stub_feature_flags(incubation_5mp_google_cloud: false) + end + + it 'returns not found' do + authorized_members.each do |authorized_member| + sign_in(authorized_member) + + get url + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + end + end +end diff --git a/spec/rubocop/cop/graphql/authorize_types_spec.rb b/spec/rubocop/cop/graphql/authorize_types_spec.rb index 6c521789e34..7aa36030526 100644 --- a/spec/rubocop/cop/graphql/authorize_types_spec.rb +++ b/spec/rubocop/cop/graphql/authorize_types_spec.rb @@ -11,7 +11,7 @@ RSpec.describe RuboCop::Cop::Graphql::AuthorizeTypes do expect_offense(<<~TYPE) module Types class AType < BaseObject - ^^^^^^^^^^^^^^^^^^^^^^^^ Add an `authorize :ability` call to the type: https://docs.gitlab.com/ee/development/api_graphql_styleguide.html#type-authorization + ^^^^^^^^^^^^^^^^^^^^^^^^ Add an `authorize :ability` call to the type: https://docs.gitlab.com/ee/development/graphql_guide/authorization.html#type-authorization field :a_thing field :another_thing end diff --git a/spec/services/google_cloud/service_accounts_service_spec.rb b/spec/services/google_cloud/service_accounts_service_spec.rb index a0d09affa72..505c623c02a 100644 --- a/spec/services/google_cloud/service_accounts_service_spec.rb +++ b/spec/services/google_cloud/service_accounts_service_spec.rb @@ -3,11 +3,11 @@ require 'spec_helper' RSpec.describe GoogleCloud::ServiceAccountsService do - let_it_be(:project) { create(:project) } - let(:service) { described_class.new(project) } describe 'find_for_project' do + let_it_be(:project) { create(:project) } + context 'when a project does not have GCP service account vars' do before do project.variables.build(key: 'blah', value: 'foo', environment_scope: 'world') @@ -21,13 +21,13 @@ RSpec.describe GoogleCloud::ServiceAccountsService do context 'when a project has GCP service account ci vars' do before do - project.variables.build(environment_scope: '*', key: 'GCP_PROJECT_ID', value: 'prj1') - project.variables.build(environment_scope: '*', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock') - project.variables.build(environment_scope: 'staging', key: 'GCP_PROJECT_ID', value: 'prj2') - project.variables.build(environment_scope: 'staging', key: 'GCP_SERVICE_ACCOUNT', value: 'mock') - project.variables.build(environment_scope: 'production', key: 'GCP_PROJECT_ID', value: 'prj3') - project.variables.build(environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT', value: 'mock') - project.variables.build(environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock') + project.variables.build(protected: true, environment_scope: '*', key: 'GCP_PROJECT_ID', value: 'prj1') + project.variables.build(protected: true, environment_scope: '*', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock') + project.variables.build(protected: true, environment_scope: 'staging', key: 'GCP_PROJECT_ID', value: 'prj2') + project.variables.build(protected: true, environment_scope: 'staging', key: 'GCP_SERVICE_ACCOUNT', value: 'mock') + project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_PROJECT_ID', value: 'prj3') + project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT', value: 'mock') + project.variables.build(protected: true, environment_scope: 'production', key: 'GCP_SERVICE_ACCOUNT_KEY', value: 'mock') project.save! end @@ -55,4 +55,55 @@ RSpec.describe GoogleCloud::ServiceAccountsService do end end end + + describe 'add_for_project' do + let_it_be(:project) { create(:project) } + + it 'saves GCP creds as project CI vars' do + service.add_for_project('env_1', 'gcp_prj_id_1', 'srv_acc_1', 'srv_acc_key_1') + service.add_for_project('env_2', 'gcp_prj_id_2', 'srv_acc_2', 'srv_acc_key_2') + + list = service.find_for_project + + aggregate_failures 'testing list of service accounts' do + expect(list.length).to eq(2) + + expect(list.first[:environment]).to eq('env_1') + expect(list.first[:gcp_project]).to eq('gcp_prj_id_1') + expect(list.first[:service_account_exists]).to eq(true) + expect(list.first[:service_account_key_exists]).to eq(true) + + expect(list.second[:environment]).to eq('env_2') + expect(list.second[:gcp_project]).to eq('gcp_prj_id_2') + expect(list.second[:service_account_exists]).to eq(true) + expect(list.second[:service_account_key_exists]).to eq(true) + end + end + + it 'replaces previously stored CI vars with new CI vars' do + service.add_for_project('env_1', 'new_project', 'srv_acc_1', 'srv_acc_key_1') + + list = service.find_for_project + + aggregate_failures 'testing list of service accounts' do + expect(list.length).to eq(2) + + # asserting that the first service account is replaced + expect(list.first[:environment]).to eq('env_1') + expect(list.first[:gcp_project]).to eq('new_project') + expect(list.first[:service_account_exists]).to eq(true) + expect(list.first[:service_account_key_exists]).to eq(true) + + expect(list.second[:environment]).to eq('env_2') + expect(list.second[:gcp_project]).to eq('gcp_prj_id_2') + expect(list.second[:service_account_exists]).to eq(true) + expect(list.second[:service_account_key_exists]).to eq(true) + end + end + + it 'underlying project CI vars must be protected' do + expect(project.variables.first.protected).to eq(true) + expect(project.variables.second.protected).to eq(true) + end + end end |