diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2017-06-07 14:30:18 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2017-06-07 14:30:18 +0300 |
commit | acd0b691a4a23bca1d90f68e2fab9004ea259a0e (patch) | |
tree | a3086d731d2eef852e25231e9a2541e3498187f4 | |
parent | cedfde549f9113865125c0bd6446416f88a9b391 (diff) | |
parent | 36ed05faf48bddd53f54971ec9b0f2c611c958d7 (diff) |
Merge branch 'feature/gb/migrate-pipeline-stages' into 'master'
Migrate pipeline stages in the database
See merge request !11714
-rw-r--r-- | db/migrate/20170525132202_create_pipeline_stages.rb | 25 | ||||
-rw-r--r-- | db/migrate/20170526185602_add_stage_id_to_ci_builds.rb | 21 | ||||
-rw-r--r-- | db/post_migrate/20170526185842_migrate_pipeline_stages.rb | 22 | ||||
-rw-r--r-- | db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb | 15 | ||||
-rw-r--r-- | db/post_migrate/20170526185921_migrate_build_stage_reference.rb | 25 | ||||
-rw-r--r-- | db/schema.rb | 19 | ||||
-rw-r--r-- | spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 | ||||
-rw-r--r-- | spec/migrations/migrate_build_stage_reference_spec.rb | 62 | ||||
-rw-r--r-- | spec/migrations/migrate_pipeline_stages_spec.rb | 56 | ||||
-rw-r--r-- | spec/services/ci/retry_build_service_spec.rb | 3 | ||||
-rw-r--r-- | spec/spec_helper.rb | 12 | ||||
-rw-r--r-- | spec/support/db_cleaner.rb | 4 | ||||
-rw-r--r-- | spec/support/migrations_helpers.rb | 29 |
13 files changed, 292 insertions, 2 deletions
diff --git a/db/migrate/20170525132202_create_pipeline_stages.rb b/db/migrate/20170525132202_create_pipeline_stages.rb new file mode 100644 index 00000000000..25656f2a2c2 --- /dev/null +++ b/db/migrate/20170525132202_create_pipeline_stages.rb @@ -0,0 +1,25 @@ +class CreatePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :ci_stages do |t| + t.integer :project_id + t.integer :pipeline_id + t.timestamps null: true + t.string :name + end + + add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade + add_concurrent_foreign_key :ci_stages, :ci_pipelines, column: :pipeline_id, on_delete: :cascade + add_concurrent_index :ci_stages, :project_id + add_concurrent_index :ci_stages, :pipeline_id + end + + def down + drop_table :ci_stages + end +end diff --git a/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb new file mode 100644 index 00000000000..d5675d5828b --- /dev/null +++ b/db/migrate/20170526185602_add_stage_id_to_ci_builds.rb @@ -0,0 +1,21 @@ +class AddStageIdToCiBuilds < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column :ci_builds, :stage_id, :integer + + add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade + add_concurrent_index :ci_builds, :stage_id + end + + def down + remove_foreign_key :ci_builds, column: :stage_id + remove_concurrent_index :ci_builds, :stage_id + + remove_column :ci_builds, :stage_id, :integer + end +end diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb new file mode 100644 index 00000000000..afd4db183c2 --- /dev/null +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -0,0 +1,22 @@ +class MigratePipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + disable_statement_timeout + + execute <<-SQL.strip_heredoc + INSERT INTO ci_stages (project_id, pipeline_id, name) + SELECT project_id, commit_id, stage FROM ci_builds + WHERE stage IS NOT NULL + AND stage_id IS NULL + AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id) + AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id) + GROUP BY project_id, commit_id, stage + ORDER BY MAX(stage_idx) + SQL + end +end diff --git a/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb b/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb new file mode 100644 index 00000000000..ec9ff33b6b7 --- /dev/null +++ b/db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb @@ -0,0 +1,15 @@ +class CreateIndexInPipelineStages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end + + def down + remove_concurrent_index(:ci_stages, [:pipeline_id, :name]) + end +end diff --git a/db/post_migrate/20170526185921_migrate_build_stage_reference.rb b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb new file mode 100644 index 00000000000..797e106cae4 --- /dev/null +++ b/db/post_migrate/20170526185921_migrate_build_stage_reference.rb @@ -0,0 +1,25 @@ +class MigrateBuildStageReference < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + disable_statement_timeout + + stage_id = Arel.sql <<-SQL.strip_heredoc + (SELECT id FROM ci_stages + WHERE ci_stages.pipeline_id = ci_builds.commit_id + AND ci_stages.name = ci_builds.stage) + SQL + + update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| + query.where(table[:stage_id].eq(nil)) + end + end + + def down + disable_statement_timeout + + update_column_in_batches(:ci_builds, :stage_id, nil) + end +end diff --git a/db/schema.rb b/db/schema.rb index ccf2672906e..400b01f73d8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170525174156) do +ActiveRecord::Schema.define(version: 20170526185921) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -234,6 +234,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do t.string "coverage_regex" t.integer "auto_canceled_by_id" t.boolean "retried" + t.integer "stage_id" end add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree @@ -243,6 +244,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree + add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree @@ -327,6 +329,18 @@ ActiveRecord::Schema.define(version: 20170525174156) do add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree + create_table "ci_stages", force: :cascade do |t| + t.integer "project_id" + t.integer "pipeline_id" + t.datetime "created_at" + t.datetime "updated_at" + t.string "name" + end + + add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree + add_index "ci_stages", ["pipeline_id"], name: "index_ci_stages_on_pipeline_id", using: :btree + add_index "ci_stages", ["project_id"], name: "index_ci_stages_on_project_id", using: :btree + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" @@ -1484,10 +1498,13 @@ ActiveRecord::Schema.define(version: 20170525174156) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify + add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify + add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade + add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 3b6bba9ffdd..de7852aa810 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -214,6 +214,7 @@ CommitStatus: - stage - trigger_request_id - stage_idx +- stage_id - tag - ref - user_id diff --git a/spec/migrations/migrate_build_stage_reference_spec.rb b/spec/migrations/migrate_build_stage_reference_spec.rb new file mode 100644 index 00000000000..80b321860c2 --- /dev/null +++ b/spec/migrations/migrate_build_stage_reference_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb') + +describe MigrateBuildStageReference, :migration do + ## + # Create test data - pipeline and CI/CD jobs. + # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } + + before do + # Create projects + # + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') + + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy') + + # Create CI/CD stages + # + stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test') + stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build') + stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy') + stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1') + stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2') + stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy') + end + + it 'correctly migrate build stage references' do + expect(jobs.where(stage_id: nil).count).to eq 8 + + migrate! + + expect(jobs.where(stage_id: nil).count).to eq 1 + + expect(jobs.find(1).stage_id).to eq 102 + expect(jobs.find(2).stage_id).to eq 102 + expect(jobs.find(3).stage_id).to eq 101 + expect(jobs.find(4).stage_id).to eq 103 + expect(jobs.find(5).stage_id).to eq 105 + expect(jobs.find(6).stage_id).to eq 104 + expect(jobs.find(7).stage_id).to eq 104 + expect(jobs.find(8).stage_id).to eq nil + end +end diff --git a/spec/migrations/migrate_pipeline_stages_spec.rb b/spec/migrations/migrate_pipeline_stages_spec.rb new file mode 100644 index 00000000000..c47f2bb8ff9 --- /dev/null +++ b/spec/migrations/migrate_pipeline_stages_spec.rb @@ -0,0 +1,56 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb') + +describe MigratePipelineStages, :migration do + ## + # Create test data - pipeline and CI/CD jobs. + # + + let(:jobs) { table(:ci_builds) } + let(:stages) { table(:ci_stages) } + let(:pipelines) { table(:ci_pipelines) } + let(:projects) { table(:projects) } + + before do + # Create projects + # + projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1') + projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2') + + # Create CI/CD pipelines + # + pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') + pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb') + + # Create CI/CD jobs + # + jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build') + jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test') + jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1') + jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2') + jobs.create!(id: 11, commit_id: 3, project_id: 456, stage_idx: 3, stage: 'deploy') + jobs.create!(id: 12, commit_id: 2, project_id: 789, stage_idx: 3, stage: 'deploy') + end + + it 'correctly migrates pipeline stages' do + expect(stages.count).to be_zero + + migrate! + + expect(stages.count).to eq 6 + expect(stages.all.pluck(:name)) + .to match_array %w[test build deploy test:1 test:2 deploy] + expect(stages.where(pipeline_id: 1).order(:id).pluck(:name)) + .to eq %w[test build deploy] + expect(stages.where(pipeline_id: 2).order(:id).pluck(:name)) + .to eq %w[test:1 test:2 deploy] + expect(stages.where(pipeline_id: 3).count).to be_zero + expect(stages.where(project_id: 789).count).to be_zero + end +end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 7254e6b357a..2bd5af25847 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do updated_at started_at finished_at queued_at erased_by erased_at auto_canceled_by].freeze + # TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR. IGNORE_ACCESSORS = %i[type lock_version target_url base_tags commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id - user_id auto_canceled_by_id retried].freeze + user_id auto_canceled_by_id retried stage_id].freeze shared_examples 'build duplication' do let(:build) do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f800c5bcb07..8b8fbf6e862 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -56,6 +56,7 @@ RSpec.configure do |config| config.include StubGitlabCalls config.include StubGitlabData config.include ApiHelpers, :api + config.include MigrationsHelpers, :migration config.infer_spec_type_from_file_location! @@ -97,6 +98,17 @@ RSpec.configure do |config| Sidekiq.redis(&:flushall) end + config.around(:example, :migration) do |example| + begin + ActiveRecord::Migrator + .migrate(migrations_paths, previous_migration.version) + + example.run + ensure + ActiveRecord::Migrator.migrate(migrations_paths) + end + end + config.around(:each, :nested_groups) do |example| example.run if Group.supports_nested_groups? end diff --git a/spec/support/db_cleaner.rb b/spec/support/db_cleaner.rb index 6f31828b825..7f5769209bb 100644 --- a/spec/support/db_cleaner.rb +++ b/spec/support/db_cleaner.rb @@ -19,6 +19,10 @@ RSpec.configure do |config| DatabaseCleaner.strategy = :truncation end + config.before(:each, :migration) do + DatabaseCleaner.strategy = :truncation + end + config.before(:each) do DatabaseCleaner.start end diff --git a/spec/support/migrations_helpers.rb b/spec/support/migrations_helpers.rb new file mode 100644 index 00000000000..91fbb4eaf48 --- /dev/null +++ b/spec/support/migrations_helpers.rb @@ -0,0 +1,29 @@ +module MigrationsHelpers + def table(name) + Class.new(ActiveRecord::Base) { self.table_name = name } + end + + def migrations_paths + ActiveRecord::Migrator.migrations_paths + end + + def table_exists?(name) + ActiveRecord::Base.connection.table_exists?(name) + end + + def migrations + ActiveRecord::Migrator.migrations(migrations_paths) + end + + def previous_migration + migrations.each_cons(2) do |previous, migration| + break previous if migration.name == described_class.name + end + end + + def migrate! + ActiveRecord::Migrator.up(migrations_paths) do |migration| + migration.name == described_class.name + end + end +end |