Add latest changes from gitlab-org/gitlab@master

7 files changed, 150 insertions, 13 deletions

diff --git a/.gitlab/ci/glfm.gitlab-ci.yml b/.gitlab/ci/glfm.gitlab-ci.yml
deleted file mode 100644
index fcc9a035c1f..00000000000
--- a/.gitlab/ci/glfm.gitlab-ci.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-glfm-verify:
-  # NOTE: We do not restrict this job to any specific subset of file changes via rules, because
-  #       there are potentially many different source files within the codebase which could
-  #       change the contents of the generated GLFM files. It is therefore safer to always
-  #       run this job to ensure that no changes are missed.
-  extends:
-    - .rspec-ee-base-pg12
-  stage: test
-  needs: ["setup-test-env"]
-  script:
-    - !reference [.base-script, script]
-    - bundle exec scripts/glfm/verify-all-generated-files-are-up-to-date.rb
diff --git a/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb
new file mode 100644
index 00000000000..16ce9bd5c85
--- /dev/null
+++ b/db/post_migrate/20220921093355_schedule_backfill_namespace_details.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class ScheduleBackfillNamespaceDetails < Gitlab::Database::Migration[2.0]
+  MIGRATION = 'BackfillNamespaceDetails'
+  INTERVAL = 2.minutes
+  BATCH_SIZE = 1_000
+  MAX_BATCH_SIZE = 10_000
+  SUB_BATCH_SIZE = 200
+
+  disable_ddl_transaction!
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :namespaces,
+      :id,
+      job_interval: INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :namespaces, :id, [])
+  end
+end
diff --git a/db/schema_migrations/20220921093355 b/db/schema_migrations/20220921093355
new file mode 100644
index 00000000000..6083a370279
--- /dev/null
+++ b/db/schema_migrations/20220921093355
@@ -0,0 +1 @@
+d7640b3756cf8cddf9f798362d6d8445a1d37092e4dff9ae263fe39a661b1e55
\ No newline at end of file
diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb
new file mode 100644
index 00000000000..b8a51b576b6
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_namespace_details.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill namespace_details for a range of namespaces
+    class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob
+      def perform
+        each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch|
+          upsert_namespace_details(sub_batch)
+        end
+      end
+
+      def upsert_namespace_details(relation)
+        connection.execute(
+          <<~SQL
+          INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id)
+            SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id
+            FROM namespaces
+            WHERE namespaces.id IN(#{relation.select(:id).to_sql})
+            AND namespaces.type <> 'Project'
+          ON CONFLICT (namespace_id) DO NOTHING;
+        SQL
+        )
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb
new file mode 100644
index 00000000000..b6282de0da6
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_namespace_details_spec.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceDetails, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:namespace_details) { table(:namespace_details) }
+
+  subject(:perform_migration) do
+    described_class.new(start_id: namespaces.minimum(:id),
+                        end_id: namespaces.maximum(:id),
+                        batch_table: :namespaces,
+                        batch_column: :id,
+                        sub_batch_size: 2,
+                        pause_ms: 0,
+                        connection: ActiveRecord::Base.connection)
+                   .perform
+  end
+
+  describe '#perform' do
+    it 'creates details for all namespaces in range' do
+      namespace1 = namespaces.create!(id: 5, name: 'test1', path: 'test1', description: "Some description1",
+                                      description_html: "Some description html1", cached_markdown_version: 4)
+      namespaces.create!(id: 6, name: 'test2', path: 'test2', type: 'Project',
+                         description: "Some description2", description_html: "Some description html2",
+                         cached_markdown_version: 4)
+      namespace3 = namespaces.create!(id: 7, name: 'test3', path: 'test3', description: "Some description3",
+                                      description_html: "Some description html3", cached_markdown_version: 4)
+      namespace4 = namespaces.create!(id: 8, name: 'test4', path: 'test4', description: "Some description3",
+                                      description_html: "Some description html4", cached_markdown_version: 4)
+      namespace_details.delete_all
+
+      expect(namespace_details.pluck(:namespace_id)).to eql []
+
+      expect { perform_migration }
+        .to change { namespace_details.pluck(:namespace_id) }.from([]).to contain_exactly(
+          namespace1.id,
+          namespace3.id,
+          namespace4.id
+        )
+
+      expect(namespace_details.find_by_namespace_id(namespace1.id)).to have_attributes(migrated_attributes(namespace1))
+      expect(namespace_details.find_by_namespace_id(namespace3.id)).to have_attributes(migrated_attributes(namespace3))
+      expect(namespace_details.find_by_namespace_id(namespace4.id)).to have_attributes(migrated_attributes(namespace4))
+    end
+  end
+
+  def migrated_attributes(namespace)
+    {
+      description: namespace.description,
+      description_html: namespace.description_html,
+      cached_markdown_version: namespace.cached_markdown_version
+    }
+  end
+end
diff --git a/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb b/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb
new file mode 100644
index 00000000000..61e4af3d10c
--- /dev/null
+++ b/spec/migrations/20220921093355_schedule_backfill_namespace_details_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe ScheduleBackfillNamespaceDetails, schema: 20220921093355 do
+  context 'when on gitlab.com' do
+    let(:background_migration) { described_class::MIGRATION }
+    let(:migration) { described_class.new }
+
+    before do
+      migration.up
+    end
+
+    describe '#up' do
+      it 'schedules background jobs for each batch of projects' do
+        expect(background_migration).to(
+          have_scheduled_batched_migration(
+            table_name: :namespaces,
+            column_name: :id,
+            interval: described_class::INTERVAL,
+            batch_size: described_class::BATCH_SIZE,
+            sub_batch_size: described_class::SUB_BATCH_SIZE
+          )
+        )
+      end
+    end
+
+    describe '#down' do
+      it 'deletes all batched migration records' do
+        migration.down
+
+        expect(described_class::MIGRATION).not_to have_scheduled_batched_migration
+      end
+    end
+  end
+end
diff --git a/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb b/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb
index d7a5bc6cc41..a4db4e25db3 100644
--- a/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb
+++ b/spec/support/shared_examples/models/concerns/cascading_namespace_setting_shared_examples.rb
@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.shared_examples 'a cascading namespace setting boolean attribute' do
-  |settings_association: :namespace_settings, settings_attribute_name:|
+  |settings_attribute_name:, settings_association: :namespace_settings|
   let_it_be_with_reload(:group) { create(:group) }
   let_it_be_with_reload(:subgroup) { create(:group, parent: group) }
   let(:group_settings) { group.send(settings_association) }