Merge branch 'backstage/gb/migrate-pipeline-stages-index' into 'master'

Migrate pipeline stages indices

Closes #43009

See merge request gitlab-org/gitlab-ce!18420

9 files changed, 112 insertions, 4 deletions

diff --git a/spec/factories/ci/stages.rb b/spec/factories/ci/stages.rb
index 25309033571..ce61e6bf759 100644
--- a/spec/factories/ci/stages.rb
+++ b/spec/factories/ci/stages.rb
@@ -21,6 +21,7 @@ FactoryBot.define do
     pipeline factory: :ci_empty_pipeline
 
     name 'test'
+    position 1
     status 'pending'
   end
 end
diff --git a/spec/factories/commit_statuses.rb b/spec/factories/commit_statuses.rb
index ce5fbc343ee..53368c64e10 100644
--- a/spec/factories/commit_statuses.rb
+++ b/spec/factories/commit_statuses.rb
@@ -2,6 +2,7 @@ FactoryBot.define do
   factory :commit_status, class: CommitStatus do
     name 'default'
     stage 'test'
+    stage_idx 0
     status 'success'
     description 'commit status'
     pipeline factory: :ci_pipeline_with_one_job
diff --git a/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb b/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb
new file mode 100644
index 00000000000..f8107dd40b9
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb
@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::MigrateStageIndex, :migration, schema: 20180420080616 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+  let(:jobs) { table(:ci_builds) }
+
+  before do
+    namespaces.create(id: 10, name: 'gitlab-org', path: 'gitlab-org')
+    projects.create!(id: 11, namespace_id: 10, name: 'gitlab', path: 'gitlab')
+    pipelines.create!(id: 12, project_id: 11, ref: 'master', sha: 'adf43c3a')
+
+    stages.create(id: 100, project_id: 11, pipeline_id: 12, name: 'build')
+    stages.create(id: 101, project_id: 11, pipeline_id: 12, name: 'test')
+
+    jobs.create!(id: 121, commit_id: 12, project_id: 11,
+                 stage_idx: 2, stage_id: 100)
+    jobs.create!(id: 122, commit_id: 12, project_id: 11,
+                 stage_idx: 2, stage_id: 100)
+    jobs.create!(id: 123, commit_id: 12, project_id: 11,
+                 stage_idx: 10, stage_id: 100)
+    jobs.create!(id: 124, commit_id: 12, project_id: 11,
+                 stage_idx: 3, stage_id: 101)
+  end
+
+  it 'correctly migrates stages indices' do
+    expect(stages.all.pluck(:position)).to all(be_nil)
+
+    described_class.new.perform(100, 101)
+
+    expect(stages.all.pluck(:position)).to eq [2, 3]
+  end
+end
diff --git a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
index dc12ba076bc..0edc3f315bb 100644
--- a/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/chain/create_spec.rb
@@ -17,7 +17,7 @@ describe Gitlab::Ci::Pipeline::Chain::Create do
 
   context 'when pipeline is ready to be saved' do
     before do
-      pipeline.stages.build(name: 'test', project: project)
+      pipeline.stages.build(name: 'test', position: 0, project: project)
 
       step.perform!
     end
diff --git a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
index eb1b285c7bd..05ce3412fd8 100644
--- a/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
+++ b/spec/lib/gitlab/ci/pipeline/seed/stage_spec.rb
@@ -24,7 +24,8 @@ describe Gitlab::Ci::Pipeline::Seed::Stage do
   describe '#attributes' do
     it 'returns hash attributes of a stage' do
       expect(subject.attributes).to be_a Hash
-      expect(subject.attributes).to include(:name, :project)
+      expect(subject.attributes)
+        .to include(:name, :position, :pipeline, :project)
     end
   end
 
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index 31141807cb2..62da967cf96 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -232,6 +232,7 @@ Ci::Stage:
 - id
 - name
 - status
+- position
 - lock_version
 - project_id
 - pipeline_id
diff --git a/spec/migrations/schedule_stages_index_migration_spec.rb b/spec/migrations/schedule_stages_index_migration_spec.rb
new file mode 100644
index 00000000000..710264da375
--- /dev/null
+++ b/spec/migrations/schedule_stages_index_migration_spec.rb
@@ -0,0 +1,35 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180420080616_schedule_stages_index_migration')
+
+describe ScheduleStagesIndexMigration, :sidekiq, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:stages) { table(:ci_stages) }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+
+    namespaces.create(id: 12, name: 'gitlab-org', path: 'gitlab-org')
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    stages.create!(id: 121, project_id: 123, pipeline_id: 1, name: 'build')
+    stages.create!(id: 122, project_id: 123, pipeline_id: 1, name: 'test')
+    stages.create!(id: 123, project_id: 123, pipeline_id: 1, name: 'deploy')
+  end
+
+  it 'schedules delayed background migrations in batches' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        expect(stages.all).to all(have_attributes(position: be_nil))
+
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 121, 121)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 122, 122)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 123, 123)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+end
diff --git a/spec/models/ci/stage_spec.rb b/spec/models/ci/stage_spec.rb
index 586d073eb5e..a00db1d2bfc 100644
--- a/spec/models/ci/stage_spec.rb
+++ b/spec/models/ci/stage_spec.rb
@@ -51,7 +51,7 @@ describe Ci::Stage, :models do
     end
   end
 
-  describe 'update_status' do
+  describe '#update_status' do
     context 'when stage objects needs to be updated' do
       before do
         create(:ci_build, :success, stage_id: stage.id)
@@ -87,4 +87,36 @@ describe Ci::Stage, :models do
       end
     end
   end
+
+  describe '#index' do
+    context 'when stage has been imported and does not have position index set' do
+      before do
+        stage.update_column(:position, nil)
+      end
+
+      context 'when stage has statuses' do
+        before do
+          create(:ci_build, :running, stage_id: stage.id, stage_idx: 10)
+        end
+
+        it 'recalculates index before updating status' do
+          expect(stage.reload.position).to be_nil
+
+          stage.update_status
+
+          expect(stage.reload.position).to eq 10
+        end
+      end
+
+      context 'when stage does not have statuses' do
+        it 'fallbacks to zero' do
+          expect(stage.reload.position).to be_nil
+
+          stage.update_status
+
+          expect(stage.reload.position).to eq 0
+        end
+      end
+    end
+  end
 end
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index 8de0bdf92e2..5bc6031388e 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -6,7 +6,9 @@ describe Ci::RetryBuildService do
   set(:pipeline) { create(:ci_pipeline, project: project) }
 
   let(:stage) do
-    Ci::Stage.create!(project: project, pipeline: pipeline, name: 'test')
+    create(:ci_stage_entity, project: project,
+                             pipeline: pipeline,
+                             name: 'test')
   end
 
   let(:build) { create(:ci_build, pipeline: pipeline, stage_id: stage.id) }