Add latest changes from gitlab-org/gitlab@master

12 files changed, 159 insertions, 16 deletions

diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb
new file mode 100644
index 00000000000..2d5231b0541
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class DesignBundlePipeline
+        include Pipeline
+
+        file_extraction_pipeline!
+        relation_name BulkImports::FileTransfer::ProjectConfig::DESIGN_BUNDLE_RELATION
+
+        def extract(_context)
+          download_service.execute
+          decompression_service.execute
+          extraction_service.execute
+
+          bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle")
+
+          BulkImports::Pipeline::ExtractedData.new(data: bundle_path)
+        end
+
+        def load(_context, bundle_path)
+          Gitlab::Utils.check_path_traversal!(bundle_path)
+          Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir])
+
+          return unless portable.lfs_enabled?
+          return unless File.exist?(bundle_path)
+          return if File.directory?(bundle_path)
+          return if File.lstat(bundle_path).symlink?
+
+          portable.design_repository.create_from_bundle(bundle_path)
+        end
+
+        def after_run(_)
+          FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir)
+        end
+
+        private
+
+        def download_service
+          BulkImports::FileDownloadService.new(
+            configuration: context.configuration,
+            relative_url: context.entity.relation_download_url_path(self.class.relation),
+            tmpdir: tmpdir,
+            filename: targz_filename
+          )
+        end
+
+        def decompression_service
+          BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename)
+        end
+
+        def extraction_service
+          BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename)
+        end
+
+        def tar_filename
+          "#{self.class.relation}.tar"
+        end
+
+        def targz_filename
+          "#{tar_filename}.gz"
+        end
+
+        def tmpdir
+          @tmpdir ||= Dir.mktmpdir('bulk_imports')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index 229df9c410d..82b1d9b427c 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -95,6 +95,10 @@ module BulkImports
             pipeline: BulkImports::Common::Pipelines::LfsObjectsPipeline,
             stage: 5
           },
+          design: {
+            pipeline: BulkImports::Projects::Pipelines::DesignBundlePipeline,
+            stage: 5
+          },
           auto_devops: {
             pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline,
             stage: 5
diff --git a/lib/generators/gitlab/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb
index 0656dfbc312..0eef6ceb539 100644
--- a/lib/generators/gitlab/usage_metric_generator.rb
+++ b/lib/generators/gitlab/usage_metric_generator.rb
@@ -15,7 +15,7 @@ module Gitlab
       redis: 'Redis'
     }.freeze
 
-    ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count).freeze
+    ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count sum).freeze
 
     source_root File.expand_path('usage_metric/templates', __dir__)
 
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 385f1e57705..c13bb1d6a9a 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -12,7 +12,7 @@ module Gitlab
       def initialize(pipeline)
         @pipeline = pipeline
 
-        super(
+        attrs = {
           object_kind: 'pipeline',
           object_attributes: hook_attrs(pipeline),
           merge_request: pipeline.merge_request && merge_request_attrs(pipeline.merge_request),
@@ -23,7 +23,13 @@ module Gitlab
             preload_builds(pipeline, :latest_builds)
             pipeline.latest_builds.map(&method(:build_hook_attrs))
           end
-        )
+        }
+
+        if pipeline.source_pipeline.present?
+          attrs[:source_pipeline] = source_pipeline_attrs(pipeline.source_pipeline)
+        end
+
+        super(attrs)
       end
 
       def with_retried_builds
@@ -72,6 +78,20 @@ module Gitlab
         }
       end
 
+      def source_pipeline_attrs(source_pipeline)
+        project = source_pipeline.source_project
+
+        {
+          project: {
+            id: project.id,
+            web_url: project.web_url,
+            path_with_namespace: project.full_path
+          },
+          job_id: source_pipeline.source_job_id,
+          pipeline_id: source_pipeline.source_pipeline_id
+        }
+      end
+
       def merge_request_attrs(merge_request)
         {
           id: merge_request.id,
diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb
index 038af570dbc..ab8b6988c3d 100644
--- a/lib/gitlab/database/migration.rb
+++ b/lib/gitlab/database/migration.rb
@@ -37,6 +37,7 @@ module Gitlab
       class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase
         include LockRetriesConcern
         include Gitlab::Database::MigrationHelpers::V2
+        include Gitlab::Database::MigrationHelpers::AnnounceDatabase
 
         # When running migrations, the `db:migrate` switches connection of
         # ActiveRecord::Base depending where the migration runs.
diff --git a/lib/gitlab/database/migration_helpers/announce_database.rb b/lib/gitlab/database/migration_helpers/announce_database.rb
new file mode 100644
index 00000000000..28710aab717
--- /dev/null
+++ b/lib/gitlab/database/migration_helpers/announce_database.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module MigrationHelpers
+      module AnnounceDatabase
+        extend ActiveSupport::Concern
+
+        def write(text = "")
+          if text.present? # announce/say
+            super("#{db_config_name}: #{text}")
+          else
+            super(text)
+          end
+        end
+
+        def db_config_name
+          @db_config_name ||= Gitlab::Database.db_config_name(connection)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
index d8d07fcaf2d..b8d1d21a0d2 100644
--- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
+++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb
@@ -21,7 +21,7 @@ module Gitlab
           end
         end
 
-        def migrate(direction)
+        def exec_migration(conn, direction)
           if unmatched_schemas.any?
             migration_skipped
             return
@@ -37,8 +37,9 @@ module Gitlab
         private
 
         def migration_skipped
-          say "Current migration is skipped since it modifies "\
-            "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'"
+          say "The migration is skipped since it modifies the schemas: #{self.class.allowed_gitlab_schemas}."
+          say "This database can only apply migrations in one of the following schemas: " \
+            "#{allowed_schemas_for_connection}."
         end
 
         def validator_class
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index a000b4509c6..48ff78cfd0f 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -18,7 +18,7 @@ module Gitlab
           UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass
 
           class << self
-            IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze
+            IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count sum).freeze
 
             private_constant :IMPLEMENTED_OPERATIONS
 
diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb
new file mode 100644
index 00000000000..6ca57864b8a
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class JiraImportsTotalImportedIssuesCountMetric < DatabaseMetric
+          operation :sum, column: :imported_issues_count
+
+          relation { JiraImportState.finished }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb
index 851aa7a50e8..91ffca4a92d 100644
--- a/lib/gitlab/usage/metrics/query.rb
+++ b/lib/gitlab/usage/metrics/query.rb
@@ -25,19 +25,19 @@ module Gitlab
           private
 
           def count(relation, column = nil)
-            raw_sql(relation, column)
+            raw_count_sql(relation, column)
           end
 
           def distinct_count(relation, column = nil)
-            raw_sql(relation, column, true)
+            raw_count_sql(relation, column, true)
           end
 
           def sum(relation, column)
-            relation.select(relation.all.table[column].sum).to_sql
+            raw_sum_sql(relation, column)
           end
 
           def estimate_batch_distinct_count(relation, column = nil)
-            raw_sql(relation, column, true)
+            raw_count_sql(relation, column, true)
           end
 
           # rubocop: disable CodeReuse/ActiveRecord
@@ -62,15 +62,23 @@ module Gitlab
           # rubocop: enable CodeReuse/ActiveRecord
 
           # rubocop: disable CodeReuse/ActiveRecord
-          def raw_sql(relation, column, distinct = false)
+          def raw_count_sql(relation, column, distinct = false)
             column ||= relation.primary_key
-            node = node_to_count(relation, column)
+            node = node_to_operate(relation, column)
 
             relation.unscope(:order).select(node.count(distinct)).to_sql
           end
           # rubocop: enable CodeReuse/ActiveRecord
 
-          def node_to_count(relation, column)
+          # rubocop: disable CodeReuse/ActiveRecord
+          def raw_sum_sql(relation, column)
+            node = node_to_operate(relation, column)
+
+            relation.unscope(:order).select(node.sum).to_sql
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def node_to_operate(relation, column)
             if join_relation?(relation) && joined_column?(column)
               table_name, column_name = column.split(".")
               Arel::Table.new(table_name)[column_name]
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index c64a6508d34..208d7c327c3 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -407,7 +407,7 @@ module Gitlab
         {
           jira_imports_total_imported_count: count(finished_jira_imports),
           jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id),
-          jira_imports_total_imported_issues_count: sum(JiraImportState.finished, :imported_issues_count)
+          jira_imports_total_imported_issues_count: add_metric('JiraImportsTotalImportedIssuesCountMetric')
         }
         # rubocop: enable UsageData/LargeTable
       end
diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb
index 2b5b3cdbb22..9dada3a3f73 100644
--- a/lib/sidebars/projects/menus/settings_menu.rb
+++ b/lib/sidebars/projects/menus/settings_menu.rb
@@ -54,7 +54,7 @@ module Sidebars
           ::Sidebars::MenuItem.new(
             title: _('Integrations'),
             link: project_settings_integrations_path(context.project),
-            active_routes: { path: %w[integrations#show services#edit] },
+            active_routes: { path: %w[integrations#index integrations#edit] },
             item_id: :integrations
           )
         end