Add latest changes from gitlab-org/gitlab@master

13 files changed, 175 insertions, 60 deletions

diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb
new file mode 100644
index 00000000000..33a24e61a3f
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class BadgesPipeline
+        include Pipeline
+
+        extractor BulkImports::Common::Extractors::RestExtractor,
+          query: BulkImports::Common::Rest::GetBadgesQuery
+
+        transformer Common::Transformers::ProhibitedAttributesTransformer
+
+        def transform(context, data)
+          return if data.blank?
+          # Project badges API returns badges of both group and project kind. To avoid creation of duplicates for the group we skip group badges when it's a project.
+          return if context.entity.project? && group_badge?(data)
+
+          {
+            name: data['name'],
+            link_url: data['link_url'],
+            image_url: data['image_url']
+          }
+        end
+
+        def load(context, data)
+          return if data.blank?
+
+          if context.entity.project?
+            context.portable.project_badges.create!(data)
+          else
+            context.portable.badges.create!(data)
+          end
+        end
+
+        private
+
+        def group_badge?(data)
+          data['kind'] == 'group'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb
index 79ffdd9a1f6..60b2ebcc552 100644
--- a/lib/bulk_imports/groups/rest/get_badges_query.rb
+++ b/lib/bulk_imports/common/rest/get_badges_query.rb
@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
 module BulkImports
-  module Groups
+  module Common
     module Rest
       module GetBadgesQuery
         extend self
 
         def to_h(context)
+          resource = context.entity.pluralized_name
           encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path)
 
           {
-            resource: ['groups', encoded_full_path, 'badges'].join('/'),
+            resource: [resource, encoded_full_path, 'badges'].join('/'),
             query: {
               page: context.tracker.next_page
             }
diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb
deleted file mode 100644
index 8569ff3f77a..00000000000
--- a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
-  module Groups
-    module Pipelines
-      class BadgesPipeline
-        include Pipeline
-
-        extractor BulkImports::Common::Extractors::RestExtractor,
-          query: BulkImports::Groups::Rest::GetBadgesQuery
-
-        transformer Common::Transformers::ProhibitedAttributesTransformer
-
-        def transform(_, data)
-          return if data.blank?
-
-          {
-            name: data['name'],
-            link_url: data['link_url'],
-            image_url: data['image_url']
-          }
-        end
-
-        def load(context, data)
-          return if data.blank?
-
-          context.group.badges.create!(data)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index 241dd428dd5..6631c212913 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -32,7 +32,7 @@ module BulkImports
             stage: 1
           },
           badges: {
-            pipeline: BulkImports::Groups::Pipelines::BadgesPipeline,
+            pipeline: BulkImports::Common::Pipelines::BadgesPipeline,
             stage: 1
           },
           boards: {
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index cc5968194a4..d95130b59e0 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -23,6 +23,10 @@ module BulkImports
             pipeline: BulkImports::Common::Pipelines::MilestonesPipeline,
             stage: 2
           },
+          badges: {
+            pipeline: BulkImports::Common::Pipelines::BadgesPipeline,
+            stage: 2
+          },
           issues: {
             pipeline: BulkImports::Projects::Pipelines::IssuesPipeline,
             stage: 3
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index 870cf25984b..68a0c15480a 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -32,12 +32,12 @@ module Gitlab
         # Models using single-type inheritance (STI) don't work with
         # reltuple count estimates. We just have to ignore them and
         # use another strategy to compute them.
-        def non_sti_models
+        def non_sti_models(models)
           models.reject { |model| sti_model?(model) }
         end
 
-        def non_sti_table_names
-          non_sti_models.map(&:table_name)
+        def non_sti_table_names(models)
+          non_sti_models(models).map(&:table_name)
         end
 
         def sti_model?(model)
@@ -45,21 +45,34 @@ module Gitlab
             model.base_class != model
         end
 
-        def table_names
-          models.map(&:table_name)
+        def table_to_model_mapping
+          @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model }
+        end
+
+        def table_to_model(table_name)
+          table_to_model_mapping[table_name]
         end
 
         def size_estimates(check_statistics: true)
-          table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model }
-
-          # Querying tuple stats only works on the primary. Due to load balancing, the
-          # easiest way to do this is to start a transaction.
-          ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases
-            get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data|
-              model = table_to_model[row.table_name]
-              data[model] = row.estimate
+          results = {}
+
+          models.group_by { |model| model.connection_db_config.name }.map do |db_name, models_for_db|
+            base_model = Gitlab::Database.database_base_models[db_name]
+            tables = non_sti_table_names(models_for_db)
+
+            # Querying tuple stats only works on the primary. Due to load balancing, the
+            # easiest way to do this is to start a transaction.
+            base_model.transaction do
+              Gitlab::Database::SharedModel.using_connection(base_model.connection) do
+                get_statistics(tables, check_statistics: check_statistics).each do |row|
+                  model = table_to_model(row.table_name)
+                  results[model] = row.estimate
+                end
+              end
             end
           end
+
+          results
         end
 
         # Generates the PostgreSQL query to return the tuples for tables
diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb
index 489bc0aacea..92c8de9aeac 100644
--- a/lib/gitlab/database/count/tablesample_count_strategy.rb
+++ b/lib/gitlab/database/count/tablesample_count_strategy.rb
@@ -61,7 +61,7 @@ module Gitlab
             #{where_clause(model)}
           SQL
 
-          rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases
+          rows = model.connection.select_all(query)
 
           Integer(rows.first['count'])
         end
diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml
index 5a553a6ef56..5695f2f1c14 100644
--- a/lib/gitlab/database/gitlab_schemas.yml
+++ b/lib/gitlab/database/gitlab_schemas.yml
@@ -261,6 +261,7 @@ issuable_severities: :gitlab_main
 issuable_slas: :gitlab_main
 issue_assignees: :gitlab_main
 issue_customer_relations_contacts: :gitlab_main
+issue_emails: :gitlab_main
 issue_email_participants: :gitlab_main
 issue_links: :gitlab_main
 issue_metrics: :gitlab_main
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index bdaf0d35a83..9bc68e9d7c9 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -258,7 +258,9 @@ module Gitlab
 
           # We keep track of the estimated number of tuples to reason later
           # about the overall progress of a migration.
-          migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
+          migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do
+            Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
+          end
           migration.save!
 
           migration
diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb
index 0ce9eebc14c..bd582d903c6 100644
--- a/lib/gitlab/database/pg_class.rb
+++ b/lib/gitlab/database/pg_class.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Database
-    class PgClass < ActiveRecord::Base
+    class PgClass < SharedModel
       self.table_name = 'pg_class'
 
       def self.for_table(relname)
diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb
index a42455aab2a..71b1d4ed8f9 100644
--- a/lib/gitlab/email/handler/service_desk_handler.rb
+++ b/lib/gitlab/email/handler/service_desk_handler.rb
@@ -32,11 +32,11 @@ module Gitlab
         def execute
           raise ProjectNotFound if project.nil?
 
-          create_issue!
+          create_issue_or_note
 
           if from_address
             add_email_participant
-            send_thank_you_email
+            send_thank_you_email unless reply_email?
           end
         end
 
@@ -82,6 +82,14 @@ module Gitlab
           project.present? && slug == project.full_path_slug
         end
 
+        def create_issue_or_note
+          if reply_email?
+            create_note_from_reply_email
+          else
+            create_issue!
+          end
+        end
+
         def create_issue!
           @issue = ::Issues::CreateService.new(
             project: project,
@@ -97,11 +105,35 @@ module Gitlab
 
           raise InvalidIssueError unless @issue.persisted?
 
+          begin
+            ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id)
+          rescue StandardError => e
+            Gitlab::ErrorTracking.log_exception(e)
+          end
+
           if service_desk_setting&.issue_template_missing?
-            create_template_not_found_note(@issue)
+            create_template_not_found_note
+          end
+        end
+
+        def issue_from_reply_to
+          strong_memoize(:issue_from_reply_to) do
+            next unless mail.in_reply_to
+
+            Issue::Email.find_by_email_message_id(mail.in_reply_to)&.issue
           end
         end
 
+        def reply_email?
+          issue_from_reply_to.present?
+        end
+
+        def create_note_from_reply_email
+          @issue = issue_from_reply_to
+
+          create_note(message_including_reply)
+        end
+
         def send_thank_you_email
           Notify.service_desk_thank_you_email(@issue.id).deliver_later
           Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email)
@@ -124,7 +156,7 @@ module Gitlab
           end
         end
 
-        def create_template_not_found_note(issue)
+        def create_template_not_found_note
           issue_template_key = service_desk_setting&.issue_template_key
 
           warning_note = <<-MD.strip_heredoc
@@ -132,15 +164,15 @@ module Gitlab
             Please check service desk settings and update the file to be used.
           MD
 
-          note_params = {
-            noteable: issue,
-            note: warning_note
-          }
+          create_note(warning_note)
+        end
 
+        def create_note(note)
           ::Notes::CreateService.new(
             project,
             User.support_bot,
-            note_params
+            noteable: @issue,
+            note: note
           ).execute
         end
 
@@ -157,6 +189,8 @@ module Gitlab
         end
 
         def add_email_participant
+          return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project)
+
           @issue.issue_email_participants.create(email: from_address)
         end
       end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index d815dd284ba..2172efba83c 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -747,6 +747,7 @@ excluded_attributes:
     - :service_desk_reply_to
     - :upvotes_count
     - :work_item_type_id
+    - :email_message_id
   merge_request: &merge_request_excluded_definition
     - :milestone_id
     - :sprint_id
diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb
index c917debd3d9..9bd2309d2b7 100644
--- a/lib/google_api/cloud_platform/client.rb
+++ b/lib/google_api/cloud_platform/client.rb
@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+require 'securerandom'
 require 'google/apis/compute_v1'
 require 'google/apis/container_v1'
 require 'google/apis/container_v1beta1'
 require 'google/apis/cloudbilling_v1'
 require 'google/apis/cloudresourcemanager_v1'
+require 'google/apis/iam_v1'
 
 module GoogleApi
   module CloudPlatform
@@ -83,6 +85,51 @@ module GoogleApi
         m[1] if m
       end
 
+      def list_projects
+        result = []
+
+        service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new
+        service.authorization = access_token
+
+        response = service.fetch_all(items: :projects) do |token|
+          service.list_projects
+        end
+
+        # Google API results are paged by default, so we need to iterate through
+        response.each do |project|
+          result.append(project)
+        end
+
+        result
+      end
+
+      def create_service_account(gcp_project_id, display_name, description)
+        name = "projects/#{gcp_project_id}"
+
+        # initialize google iam service
+        service = Google::Apis::IamV1::IamService.new
+        service.authorization = access_token
+
+        # generate account id
+        random_account_id = "gitlab-" + SecureRandom.hex(11)
+
+        body_params = { account_id: random_account_id,
+                        service_account: { display_name: display_name,
+                                           description: description } }
+
+        request_body = Google::Apis::IamV1::CreateServiceAccountRequest.new(**body_params)
+        service.create_service_account(name, request_body)
+      end
+
+      def create_service_account_key(gcp_project_id, service_account_id)
+        service = Google::Apis::IamV1::IamService.new
+        service.authorization = access_token
+
+        name = "projects/#{gcp_project_id}/serviceAccounts/#{service_account_id}"
+        request_body = Google::Apis::IamV1::CreateServiceAccountKeyRequest.new
+        service.create_service_account_key(name, request_body)
+      end
+
       private
 
       def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons)