Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42

5 files changed, 147 insertions, 100 deletions

diff --git a/spec/db/development/create_work_item_hierarchy_restrictions_spec.rb b/spec/db/development/create_work_item_hierarchy_restrictions_spec.rb
new file mode 100644
index 00000000000..0e60ecd08c0
--- /dev/null
+++ b/spec/db/development/create_work_item_hierarchy_restrictions_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Create work item hierarchy restrictions in development', feature_category: :portfolio_management do
+  subject { load Rails.root.join('db/fixtures/development/50_create_work_item_hierarchy_restrictions.rb') }
+
+  it_behaves_like 'work item hierarchy restrictions importer'
+end
diff --git a/spec/db/docs_spec.rb b/spec/db/docs_spec.rb
index ad3705c3dbe..6cfff725988 100644
--- a/spec/db/docs_spec.rb
+++ b/spec/db/docs_spec.rb
@@ -2,108 +2,95 @@
 
 require 'spec_helper'
 
-RSpec.describe 'Database Documentation' do
-  context 'for each table' do
-    # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834
-    let(:database_base_models) { Gitlab::Database.database_base_models.select { |k, _| k != 'geo' } }
-
-    let(:all_tables) do
-      database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq
-    end
-
-    let(:metadata_required_fields) do
-      %i(
-        feature_categories
-        table_name
-      )
-    end
+RSpec.shared_examples 'validate dictionary' do |objects, directory_path, required_fields|
+  context 'for each object' do
+    let(:directory_path) {  directory_path }
 
     let(:metadata_allowed_fields) do
-      metadata_required_fields + %i(
+      required_fields + %i[
         classes
         description
         introduced_by_url
         milestone
-      )
+        gitlab_schema
+      ]
     end
 
     let(:metadata) do
-      all_tables.each_with_object({}) do |table_name, hash|
-        next unless File.exist?(table_metadata_file_path(table_name))
+      objects.each_with_object({}) do |object_name, hash|
+        next unless File.exist?(object_metadata_file_path(object_name))
 
-        hash[table_name] ||= load_table_metadata(table_name)
+        hash[object_name] ||= load_object_metadata(required_fields, object_name)
       end
     end
 
-    let(:tables_without_metadata) do
-      all_tables.reject { |t| metadata.has_key?(t) }
+    let(:objects_without_metadata) do
+      objects.reject { |t| metadata.has_key?(t) }
     end
 
-    let(:tables_without_valid_metadata) do
+    let(:objects_without_valid_metadata) do
       metadata.select { |_, t| t.has_key?(:error) }.keys
     end
 
-    let(:tables_with_disallowed_fields) do
+    let(:objects_with_disallowed_fields) do
       metadata.select { |_, t| t.has_key?(:disallowed_fields) }.keys
     end
 
-    let(:tables_with_missing_required_fields) do
+    let(:objects_with_missing_required_fields) do
       metadata.select { |_, t| t.has_key?(:missing_required_fields) }.keys
     end
 
     it 'has a metadata file' do
-      expect(tables_without_metadata).to be_empty, multiline_error(
+      expect(objects_without_metadata).to be_empty, multiline_error(
         'Missing metadata files',
-        tables_without_metadata.map { |t| "  #{table_metadata_file(t)}" }
+        objects_without_metadata.map { |t| "  #{object_metadata_file(t)}" }
       )
     end
 
     it 'has a valid metadata file' do
-      expect(tables_without_valid_metadata).to be_empty, table_metadata_errors(
+      expect(objects_without_valid_metadata).to be_empty, object_metadata_errors(
         'Table metadata files with errors',
         :error,
-        tables_without_valid_metadata
+        objects_without_valid_metadata
       )
     end
 
     it 'has a valid metadata file with allowed fields' do
-      expect(tables_with_disallowed_fields).to be_empty, table_metadata_errors(
+      expect(objects_with_disallowed_fields).to be_empty, object_metadata_errors(
         'Table metadata files with disallowed fields',
         :disallowed_fields,
-        tables_with_disallowed_fields
+        objects_with_disallowed_fields
       )
     end
 
     it 'has a valid metadata file without missing fields' do
-      expect(tables_with_missing_required_fields).to be_empty, table_metadata_errors(
+      expect(objects_with_missing_required_fields).to be_empty, object_metadata_errors(
         'Table metadata files with missing fields',
         :missing_required_fields,
-        tables_with_missing_required_fields
+        objects_with_missing_required_fields
       )
     end
   end
 
   private
 
-  def table_metadata_file(table_name)
-    File.join('db', 'docs', "#{table_name}.yml")
+  def object_metadata_file(object_name)
+    File.join(directory_path, "#{object_name}.yml")
   end
 
-  def table_metadata_file_path(table_name)
-    Rails.root.join(table_metadata_file(table_name))
+  def object_metadata_file_path(object_name)
+    Rails.root.join(object_metadata_file(object_name))
   end
 
-  def load_table_metadata(table_name)
+  def load_object_metadata(required_fields, object_name)
     result = {}
     begin
-      result[:metadata] = YAML.safe_load(File.read(table_metadata_file_path(table_name))).deep_symbolize_keys
+      result[:metadata] = YAML.safe_load(File.read(object_metadata_file_path(object_name))).deep_symbolize_keys
 
       disallowed_fields = (result[:metadata].keys - metadata_allowed_fields)
-      unless disallowed_fields.empty?
-        result[:disallowed_fields] = "fields not allowed: #{disallowed_fields.join(', ')}"
-      end
+      result[:disallowed_fields] = "fields not allowed: #{disallowed_fields.join(', ')}" unless disallowed_fields.empty?
 
-      missing_required_fields = (metadata_required_fields - result[:metadata].reject { |_, v| v.blank? }.keys)
+      missing_required_fields = (required_fields - result[:metadata].reject { |_, v| v.blank? }.keys)
       unless missing_required_fields.empty?
         result[:missing_required_fields] = "missing required fields: #{missing_required_fields.join(', ')}"
       end
@@ -113,11 +100,12 @@ RSpec.describe 'Database Documentation' do
     result
   end
 
-  def table_metadata_errors(title, field, tables)
-    lines = tables.map do |table_name|
+  # rubocop:disable Naming/HeredocDelimiterNaming
+  def object_metadata_errors(title, field, objects)
+    lines = objects.map do |object_name|
       <<~EOM
-        #{table_metadata_file(table_name)}
-          #{metadata[table_name][field]}
+        #{object_metadata_file(object_name)}
+          #{metadata[object_name][field]}
       EOM
     end
 
@@ -131,4 +119,23 @@ RSpec.describe 'Database Documentation' do
       #{lines.join("\n")}
     EOM
   end
+  # rubocop:enable Naming/HeredocDelimiterNaming
+end
+
+RSpec.describe 'Views documentation', feature_category: :database do
+  database_base_models = Gitlab::Database.database_base_models.select { |k, _| k != 'geo' }
+  views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq
+  directory_path = File.join('db', 'docs', 'views')
+  required_fields = %i[feature_categories view_name gitlab_schema]
+
+  include_examples 'validate dictionary', views, directory_path, required_fields
+end
+
+RSpec.describe 'Tables documentation', feature_category: :database do
+  database_base_models = Gitlab::Database.database_base_models.select { |k, _| k != 'geo' }
+  tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq
+  directory_path = File.join('db', 'docs')
+  required_fields = %i[feature_categories table_name gitlab_schema]
+
+  include_examples 'validate dictionary', tables, directory_path, required_fields
 end
diff --git a/spec/db/migration_spec.rb b/spec/db/migration_spec.rb
index 7751bfd989d..b5f6192233f 100644
--- a/spec/db/migration_spec.rb
+++ b/spec/db/migration_spec.rb
@@ -8,6 +8,7 @@ RSpec.describe 'Migrations Validation' do
   # The range describes the timestamps that given migration helper can be used
   let(:all_migration_classes) do
     {
+      2022_12_01_02_15_00.. => Gitlab::Database::Migration[2.1],
       2022_01_26_21_06_58.. => Gitlab::Database::Migration[2.0],
       2021_09_01_15_33_24..2022_04_25_12_06_03 => Gitlab::Database::Migration[1.0],
       2021_05_31_05_39_16..2021_09_01_15_33_24 => ActiveRecord::Migration[6.1],
diff --git a/spec/db/production/create_work_item_hierarchy_restrictions_spec.rb b/spec/db/production/create_work_item_hierarchy_restrictions_spec.rb
new file mode 100644
index 00000000000..5b47d88d71a
--- /dev/null
+++ b/spec/db/production/create_work_item_hierarchy_restrictions_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Create work item hierarchy restrictions in production', feature_category: :portfolio_management do
+  subject { load Rails.root.join('db/fixtures/production/020_create_work_item_hierarchy_restrictions.rb') }
+
+  it_behaves_like 'work item hierarchy restrictions importer'
+end
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index ad49a763361..9e23cca7c3f 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -6,12 +6,11 @@ require Rails.root.join('ee', 'spec', 'db', 'schema_support') if Gitlab.ee?
 RSpec.describe 'Database schema' do
   prepend_mod_with('DB::SchemaSupport')
 
-  let(:connection) { ActiveRecord::Base.connection }
   let(:tables) { connection.tables }
   let(:columns_name_with_jsonb) { retrieve_columns_name_with_jsonb }
 
   IGNORED_INDEXES_ON_FKS = {
-    issues: %w[work_item_type_id]
+    slack_integrations_scopes: %w[slack_api_scope_id]
   }.with_indifferent_access.freeze
 
   TABLE_PARTITIONS = %w[ci_builds_metadata].freeze
@@ -33,15 +32,27 @@ RSpec.describe 'Database schema' do
     boards: %w[milestone_id iteration_id],
     chat_names: %w[chat_id team_id user_id],
     chat_teams: %w[team_id],
+    ci_build_needs: %w[partition_id],
+    ci_build_pending_states: %w[partition_id],
+    ci_build_report_results: %w[partition_id],
+    ci_build_trace_chunks: %w[partition_id],
+    ci_build_trace_metadata: %w[partition_id],
     ci_builds: %w[erased_by_id trigger_request_id partition_id],
+    ci_builds_runner_session: %w[partition_id],
     p_ci_builds_metadata: %w[partition_id],
     ci_job_artifacts: %w[partition_id],
+    ci_job_variables: %w[partition_id],
     ci_namespace_monthly_usages: %w[namespace_id],
+    ci_pending_builds: %w[partition_id],
     ci_pipeline_variables: %w[partition_id],
     ci_pipelines: %w[partition_id],
+    ci_resources: %w[partition_id],
     ci_runner_projects: %w[runner_id],
+    ci_running_builds: %w[partition_id],
+    ci_sources_pipelines: %w[partition_id source_partition_id],
     ci_stages: %w[partition_id],
     ci_trigger_requests: %w[commit_id],
+    ci_unit_test_failures: %w[partition_id],
     cluster_providers_aws: %w[security_group_id vpc_id access_key_id],
     cluster_providers_gcp: %w[gcp_project_id operation_id],
     compliance_management_frameworks: %w[group_id],
@@ -109,56 +120,62 @@ RSpec.describe 'Database schema' do
   }.with_indifferent_access.freeze
 
   context 'for table' do
-    (ActiveRecord::Base.connection.tables - TABLE_PARTITIONS).sort.each do |table|
-      describe table do
-        let(:indexes) { connection.indexes(table) }
-        let(:columns) { connection.columns(table) }
-        let(:foreign_keys) { connection.foreign_keys(table) }
-        let(:loose_foreign_keys) { Gitlab::Database::LooseForeignKeys.definitions.group_by(&:from_table).fetch(table, []) }
-        let(:all_foreign_keys) { foreign_keys + loose_foreign_keys }
-        # take the first column in case we're using a composite primary key
-        let(:primary_key_column) { Array(connection.primary_key(table)).first }
-
-        context 'all foreign keys' do
-          # for index to be effective, the FK constraint has to be at first place
-          it 'are indexed' do
-            first_indexed_column = indexes.filter_map do |index|
-              columns = index.columns
-
-              # In cases of complex composite indexes, a string is returned eg:
-              # "lower((extern_uid)::text), group_id"
-              columns = columns.split(',') if columns.is_a?(String)
-              column = columns.first.chomp
-
-              # A partial index is not suitable for a foreign key column, unless
-              # the only condition is for the presence of the foreign key itself
-              column if index.where.nil? || index.where == "(#{column} IS NOT NULL)"
+    Gitlab::Database::EachDatabase.each_database_connection do |connection, _|
+      schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection)
+      (connection.tables - TABLE_PARTITIONS).sort.each do |table|
+        table_schema = Gitlab::Database::GitlabSchema.table_schema(table)
+        next unless schemas_for_connection.include?(table_schema)
+
+        describe table do
+          let(:indexes) { connection.indexes(table) }
+          let(:columns) { connection.columns(table) }
+          let(:foreign_keys) { connection.foreign_keys(table) }
+          let(:loose_foreign_keys) { Gitlab::Database::LooseForeignKeys.definitions.group_by(&:from_table).fetch(table, []) }
+          let(:all_foreign_keys) { foreign_keys + loose_foreign_keys }
+          # take the first column in case we're using a composite primary key
+          let(:primary_key_column) { Array(connection.primary_key(table)).first }
+
+          context 'all foreign keys' do
+            # for index to be effective, the FK constraint has to be at first place
+            it 'are indexed' do
+              first_indexed_column = indexes.filter_map do |index|
+                columns = index.columns
+
+                # In cases of complex composite indexes, a string is returned eg:
+                # "lower((extern_uid)::text), group_id"
+                columns = columns.split(',') if columns.is_a?(String)
+                column = columns.first.chomp
+
+                # A partial index is not suitable for a foreign key column, unless
+                # the only condition is for the presence of the foreign key itself
+                column if index.where.nil? || index.where == "(#{column} IS NOT NULL)"
+              end
+              foreign_keys_columns = all_foreign_keys.map(&:column)
+              required_indexed_columns = foreign_keys_columns - ignored_index_columns(table)
+
+              # Add the primary key column to the list of indexed columns because
+              # postgres and mysql both automatically create an index on the primary
+              # key. Also, the rails connection.indexes() method does not return
+              # automatically generated indexes (like the primary key index).
+              first_indexed_column.push(primary_key_column)
+
+              expect(first_indexed_column.uniq).to include(*required_indexed_columns)
             end
-            foreign_keys_columns = all_foreign_keys.map(&:column)
-            required_indexed_columns = foreign_keys_columns - ignored_index_columns(table)
-
-            # Add the primary key column to the list of indexed columns because
-            # postgres and mysql both automatically create an index on the primary
-            # key. Also, the rails connection.indexes() method does not return
-            # automatically generated indexes (like the primary key index).
-            first_indexed_column.push(primary_key_column)
-
-            expect(first_indexed_column.uniq).to include(*required_indexed_columns)
           end
-        end
 
-        context 'columns ending with _id' do
-          let(:column_names) { columns.map(&:name) }
-          let(:column_names_with_id) { column_names.select { |column_name| column_name.ends_with?('_id') } }
-          let(:foreign_keys_columns) { all_foreign_keys.map(&:column).uniq } # we can have FK and loose FK present at the same time
-          let(:ignored_columns) { ignored_fk_columns(table) }
+          context 'columns ending with _id' do
+            let(:column_names) { columns.map(&:name) }
+            let(:column_names_with_id) { column_names.select { |column_name| column_name.ends_with?('_id') } }
+            let(:foreign_keys_columns) { all_foreign_keys.map(&:column).uniq } # we can have FK and loose FK present at the same time
+            let(:ignored_columns) { ignored_fk_columns(table) }
 
-          it 'do have the foreign keys' do
-            expect(column_names_with_id - ignored_columns).to match_array(foreign_keys_columns)
-          end
+            it 'do have the foreign keys' do
+              expect(column_names_with_id - ignored_columns).to match_array(foreign_keys_columns)
+            end
 
-          it 'and having foreign key are not in the ignore list' do
-            expect(ignored_columns).to match_array(ignored_columns - foreign_keys)
+            it 'and having foreign key are not in the ignore list' do
+              expect(ignored_columns).to match_array(ignored_columns - foreign_keys)
+            end
           end
         end
       end
@@ -225,7 +242,8 @@ RSpec.describe 'Database schema' do
     "Packages::Composer::Metadatum" => %w[composer_json],
     "RawUsageData" => %w[payload], # Usage data payload changes often, we cannot use one schema
     "Releases::Evidence" => %w[summary],
-    "Vulnerabilities::Finding::Evidence" => %w[data] # Validation work in progress
+    "Vulnerabilities::Finding::Evidence" => %w[data], # Validation work in progress
+    "EE::Gitlab::BackgroundMigration::FixSecurityScanStatuses::SecurityScan" => %w[info] # This is a migration model
   }.freeze
 
   # We are skipping GEO models for now as it adds up complexity
@@ -275,13 +293,16 @@ RSpec.describe 'Database schema' do
 
   context 'primary keys' do
     it 'expects every table to have a primary key defined' do
-      connection = ActiveRecord::Base.connection
+      Gitlab::Database::EachDatabase.each_database_connection do |connection, _|
+        schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection)
 
-      problematic_tables = connection.tables.select do |table|
-        !connection.primary_key(table).present?
-      end.map(&:to_sym)
+        problematic_tables = connection.tables.select do |table|
+          table_schema = Gitlab::Database::GitlabSchema.table_schema(table)
+          schemas_for_connection.include?(table_schema) && !connection.primary_key(table).present?
+        end.map(&:to_sym)
 
-      expect(problematic_tables).to be_empty
+        expect(problematic_tables).to be_empty
+      end
     end
   end