Add latest changes from gitlab-org/gitlab@master

3 files changed, 155 insertions, 1 deletions

diff --git a/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb
new file mode 100644
index 00000000000..23c510720c0
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill the following columns on the namespace_root_storage_statistics table:
+    #   - public_forks_storage_size
+    #   - internal_forks_storage_size
+    #   - private_forks_storage_size
+    class BackfillRootStorageStatisticsForkStorageSizes < BatchedMigrationJob
+      operation_name :backfill_root_storage_statistics_fork_sizes
+      feature_category :consumables_cost_management
+
+      VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS = {
+        0 => :private_forks_storage_size,
+        10 => :internal_forks_storage_size,
+        20 => :public_forks_storage_size
+      }.freeze
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.each do |root_storage_statistics|
+            next if has_fork_data?(root_storage_statistics)
+
+            namespace_id = root_storage_statistics.namespace_id
+
+            namespace_type = execute("SELECT type FROM namespaces WHERE id = #{namespace_id}").first&.fetch('type')
+
+            next if namespace_type.nil?
+
+            sql = if user_namespace?(namespace_type)
+                    user_namespace_sql(namespace_id)
+                  else
+                    group_namespace_sql(namespace_id)
+                  end
+
+            stats = execute(sql)
+              .map { |h| { h['projects_visibility_level'] => h['sum_project_statistics_storage_size'] } }
+              .reduce({}) { |memo, h| memo.merge(h) }
+              .transform_keys { |k| VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS[k] }
+
+            root_storage_statistics.update!(stats)
+          end
+        end
+      end
+
+      def has_fork_data?(root_storage_statistics)
+        root_storage_statistics.public_forks_storage_size != 0 ||
+          root_storage_statistics.internal_forks_storage_size != 0 ||
+          root_storage_statistics.private_forks_storage_size != 0
+      end
+
+      def user_namespace?(type)
+        type.nil? || type == 'User' || !(type == 'Group' || type == 'Project')
+      end
+
+      def execute(sql)
+        ::ApplicationRecord.connection.execute(sql)
+      end
+
+      def user_namespace_sql(namespace_id)
+        <<~SQL
+          SELECT
+            SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size,
+            "projects"."visibility_level" AS projects_visibility_level
+          FROM
+            "projects"
+            INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id"
+            INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id"
+            INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id"
+          WHERE
+            "projects"."namespace_id" = #{namespace_id}
+            AND (fork_networks.root_project_id != projects.id)
+          GROUP BY "projects"."visibility_level"
+        SQL
+      end
+
+      def group_namespace_sql(namespace_id)
+        <<~SQL
+          SELECT
+            SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size,
+            "projects"."visibility_level" AS projects_visibility_level
+          FROM
+            "projects"
+            INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id"
+            INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id"
+            INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id"
+          WHERE
+            "projects"."namespace_id" IN (
+              SELECT namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id
+              FROM "namespaces"
+              WHERE "namespaces"."type" = 'Group' AND (traversal_ids @> ('{#{namespace_id}}'))
+            )
+            AND (fork_networks.root_project_id != projects.id)
+          GROUP BY "projects"."visibility_level"
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/convert_feature_category_to_group_label.rb b/lib/gitlab/database/convert_feature_category_to_group_label.rb
new file mode 100644
index 00000000000..5a4599312ba
--- /dev/null
+++ b/lib/gitlab/database/convert_feature_category_to_group_label.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class ConvertFeatureCategoryToGroupLabel
+      STAGES_URL = 'https://gitlab.com/gitlab-com/www-gitlab-com/-/raw/master/data/stages.yml'
+
+      def initialize(feature_category)
+        @feature_category = feature_category
+      end
+
+      def execute
+        feature_categories_map[feature_category]
+      end
+
+      private
+
+      attr_reader :feature_category
+
+      def stages
+        response = Gitlab::HTTP.get(STAGES_URL)
+
+        YAML.safe_load(response) if response.success?
+      end
+
+      def feature_categories_map
+        stages['stages'].each_with_object({}) do |(_, stage), result|
+          stage['groups'].each do |group_name, group|
+            group['categories'].each do |category|
+              result[category] = "group::#{group_name.sub('_', ' ')}"
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb
index 32118f1f60d..524c114810f 100644
--- a/lib/gitlab/database/schema_validation/track_inconsistency.rb
+++ b/lib/gitlab/database/schema_validation/track_inconsistency.rb
@@ -41,7 +41,7 @@ module Gitlab
             title: issue_title,
             description: description,
             issue_type: 'issue',
-            labels: %w[database database-inconsistency-report]
+            labels: default_labels + group_labels
           }
         end
 
@@ -84,6 +84,24 @@ module Gitlab
           MSG
         end
 
+        def group_labels
+          dictionary = YAML.safe_load(File.read(table_file_path))
+
+          dictionary['feature_categories'].to_a.filter_map do |feature_category|
+            Gitlab::Database::ConvertFeatureCategoryToGroupLabel.new(feature_category).execute
+          end
+        rescue Errno::ENOENT
+          []
+        end
+
+        def default_labels
+          %w[database database-inconsistency-report type::maintenance severity::4]
+        end
+
+        def table_file_path
+          Rails.root.join(Gitlab::Database::GitlabSchema.dictionary_paths.first, "#{inconsistency.table_name}.yml")
+        end
+
         def schema_inconsistency_model
           Gitlab::Database::SchemaValidation::SchemaInconsistency
         end