diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/bulk_imports/common/pipelines/badges_pipeline.rb | 44 | ||||
-rw-r--r-- | lib/bulk_imports/common/rest/get_badges_query.rb (renamed from lib/bulk_imports/groups/rest/get_badges_query.rb) | 5 | ||||
-rw-r--r-- | lib/bulk_imports/groups/pipelines/badges_pipeline.rb | 32 | ||||
-rw-r--r-- | lib/bulk_imports/groups/stage.rb | 2 | ||||
-rw-r--r-- | lib/bulk_imports/projects/stage.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/database/count/reltuples_count_strategy.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/database/count/tablesample_count_strategy.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/database/gitlab_schemas.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/database/migrations/background_migration_helpers.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/database/pg_class.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/email/handler/service_desk_handler.rb | 52 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 1 | ||||
-rw-r--r-- | lib/google_api/cloud_platform/client.rb | 47 |
13 files changed, 175 insertions, 60 deletions
diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb new file mode 100644 index 00000000000..33a24e61a3f --- /dev/null +++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class BadgesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::RestExtractor, + query: BulkImports::Common::Rest::GetBadgesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + return if data.blank? + # Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project. + return if context.entity.project? && group_badge?(data) + + { + name: data['name'], + link_url: data['link_url'], + image_url: data['image_url'] + } + end + + def load(context, data) + return if data.blank? + + if context.entity.project? + context.portable.project_badges.create!(data) + else + context.portable.badges.create!(data) + end + end + + private + + def group_badge?(data) + data['kind'] == 'group' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb index 79ffdd9a1f6..60b2ebcc552 100644 --- a/lib/bulk_imports/groups/rest/get_badges_query.rb +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Rest module GetBadgesQuery extend self def to_h(context) + resource = context.entity.pluralized_name encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) { - resource: ['groups', encoded_full_path, 'badges'].join('/'), + resource: [resource, encoded_full_path, 'badges'].join('/'), query: { page: context.tracker.next_page } diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb deleted file mode 100644 index 8569ff3f77a..00000000000 --- a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Pipelines - class BadgesPipeline - include Pipeline - - extractor BulkImports::Common::Extractors::RestExtractor, - query: BulkImports::Groups::Rest::GetBadgesQuery - - transformer Common::Transformers::ProhibitedAttributesTransformer - - def transform(_, data) - return if data.blank? - - { - name: data['name'], - link_url: data['link_url'], - image_url: data['image_url'] - } - end - - def load(context, data) - return if data.blank? - - context.group.badges.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 241dd428dd5..6631c212913 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -32,7 +32,7 @@ module BulkImports stage: 1 }, badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, stage: 1 }, boards: { diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index cc5968194a4..d95130b59e0 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -23,6 +23,10 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 2 }, + badges: { + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 870cf25984b..68a0c15480a 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -32,12 +32,12 @@ module Gitlab # Models using single-type inheritance (STI) don't work with # reltuple count estimates. We just have to ignore them and # use another strategy to compute them. - def non_sti_models + def non_sti_models(models) models.reject { |model| sti_model?(model) } end - def non_sti_table_names - non_sti_models.map(&:table_name) + def non_sti_table_names(models) + non_sti_models(models).map(&:table_name) end def sti_model?(model) @@ -45,21 +45,34 @@ module Gitlab model.base_class != model end - def table_names - models.map(&:table_name) + def table_to_model_mapping + @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + end + + def table_to_model(table_name) + table_to_model_mapping[table_name] end def size_estimates(check_statistics: true) - table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } - - # Querying tuple stats only works on the primary. Due to load balancing, the - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases - get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| - model = table_to_model[row.table_name] - data[model] = row.estimate + results = {} + + models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db| + base_model = Gitlab::Database.database_base_models[db_name] + tables = non_sti_table_names(models_for_db) + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + base_model.transaction do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + get_statistics(tables, check_statistics: check_statistics).each do |row| + model = table_to_model(row.table_name) + results[model] = row.estimate + end + end end end + + results end # Generates the PostgreSQL query to return the tuples for tables diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index 489bc0aacea..92c8de9aeac 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -61,7 +61,7 @@ module Gitlab #{where_clause(model)} SQL - rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases + rows = model.connection.select_all(query) Integer(rows.first['count']) end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5a553a6ef56..5695f2f1c14 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -261,6 +261,7 @@ issuable_severities: :gitlab_main issuable_slas: :gitlab_main issue_assignees: :gitlab_main issue_customer_relations_contacts: :gitlab_main +issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index bdaf0d35a83..9bc68e9d7c9 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -258,7 +258,9 @@ module Gitlab # We keep track of the estimated number of tuples to reason later # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end migration.save! migration diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb index 0ce9eebc14c..bd582d903c6 100644 --- a/lib/gitlab/database/pg_class.rb +++ b/lib/gitlab/database/pg_class.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PgClass < ActiveRecord::Base + class PgClass < SharedModel self.table_name = 'pg_class' def self.for_table(relname) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index a42455aab2a..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,11 +32,11 @@ module Gitlab def execute raise ProjectNotFound if project.nil? - create_issue! + create_issue_or_note if from_address add_email_participant - send_thank_you_email + send_thank_you_email unless reply_email? end end @@ -82,6 +82,14 @@ module Gitlab project.present? && slug == project.full_path_slug end + def create_issue_or_note + if reply_email? + create_note_from_reply_email + else + create_issue! + end + end + def create_issue! @issue = ::Issues::CreateService.new( project: project, @@ -97,11 +105,35 @@ module Gitlab raise InvalidIssueError unless @issue.persisted? + begin + ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception(e) + end + if service_desk_setting&.issue_template_missing? - create_template_not_found_note(@issue) + create_template_not_found_note + end + end + + def issue_from_reply_to + strong_memoize(:issue_from_reply_to) do + next unless mail.in_reply_to + + Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue end end + def reply_email? + issue_from_reply_to.present? + end + + def create_note_from_reply_email + @issue = issue_from_reply_to + + create_note(message_including_reply) + end + def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) @@ -124,7 +156,7 @@ module Gitlab end end - def create_template_not_found_note(issue) + def create_template_not_found_note issue_template_key = service_desk_setting&.issue_template_key warning_note = <<-MD.strip_heredoc @@ -132,15 +164,15 @@ module Gitlab Please check service desk settings and update the file to be used. MD - note_params = { - noteable: issue, - note: warning_note - } + create_note(warning_note) + end + def create_note(note) ::Notes::CreateService.new( project, User.support_bot, - note_params + noteable: @issue, + note: note ).execute end @@ -157,6 +189,8 @@ module Gitlab end def add_email_participant + return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + @issue.issue_email_participants.create(email: from_address) end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d815dd284ba..2172efba83c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -747,6 +747,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id + - :email_message_id merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index c917debd3d9..9bd2309d2b7 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true +require 'securerandom' require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/container_v1beta1' require 'google/apis/cloudbilling_v1' require 'google/apis/cloudresourcemanager_v1' +require 'google/apis/iam_v1' module GoogleApi module CloudPlatform @@ -83,6 +85,51 @@ module GoogleApi m[1] if m end + def list_projects + result = [] + + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + response = service.fetch_all(items: :projects) do |token| + service.list_projects + end + + # Google API results are paged by default, so we need to iterate through + response.each do |project| + result.append(project) + end + + result + end + + def create_service_account(gcp_project_id, display_name, description) + name = "projects/#{gcp_project_id}" + + # initialize google iam service + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + # generate account id + random_account_id = "gitlab-" + SecureRandom.hex(11) + + body_params = { account_id: random_account_id, + service_account: { display_name: display_name, + description: description } } + + request_body = Google::Apis::IamV1::CreateServiceAccountRequest.new(**body_params) + service.create_service_account(name, request_body) + end + + def create_service_account_key(gcp_project_id, service_account_id) + service = Google::Apis::IamV1::IamService.new + service.authorization = access_token + + name = "projects/#{gcp_project_id}/serviceAccounts/#{service_account_id}" + request_body = Google::Apis::IamV1::CreateServiceAccountKeyRequest.new + service.create_service_account_key(name, request_body) + end + private def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) |