From b21ee2ee36e1aaddbe0b3541a8cac5f117143b66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 11:29:04 +0200 Subject: Add initial stage_id background migration files --- ...0170628080858_migrate_stage_id_reference_in_background.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb new file mode 100644 index 00000000000..2eaa798d0aa --- /dev/null +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -0,0 +1,12 @@ +class MigrateStageIdReferenceInBackground < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + end + + def down + # noop + end +end -- cgit v1.2.3 From 98992c4e4b3231a99eb5ff17c44e96fe79a6cff2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:01:52 +0200 Subject: Add initial build stage_id ref background migration --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 2eaa798d0aa..44bac4a8cc7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -3,7 +3,17 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration DOWNTIME = false + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end + def up + Build.find_each do |build| + BackgroundMigrationWorker + .perform_async('MigrateBuildStageIdReference', [build.id]) + end end def down -- cgit v1.2.3 From 02bb40e2acd7b1838e47e1a2f8b9288e42e6ca53 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:21:25 +0200 Subject: Find builds that require a migration in batches --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 44bac4a8cc7..6b326bc0b69 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -2,6 +2,8 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers DOWNTIME = false + BATCH_SIZE = 10000 + MIGRATION = 'MigrateBuildStageIdReference'.freeze disable_ddl_transaction! @@ -10,9 +12,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_each do |build| - BackgroundMigrationWorker - .perform_async('MigrateBuildStageIdReference', [build.id]) + Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } + + BackgroundMigrationWorker.perform_bulk(*migrations) end end -- cgit v1.2.3 From 5292eb651e1e3595e409a4c216eb0be3445a9319 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 12:23:00 +0200 Subject: Schedule background migration only when it is needed --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 6b326bc0b69..bfeb09f6da1 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,11 +12,13 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.find_in_batches(batch_size: BATCH_SIZE).with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } + Build.where(stage_id: nil) + .find_in_batches(batch_size: BATCH_SIZE) + .with_index do |builds, batch| + migrations = builds.map { |build| [MIGRATION, [build.id]] } - BackgroundMigrationWorker.perform_bulk(*migrations) - end + BackgroundMigrationWorker.perform_bulk(*migrations) + end end def down -- cgit v1.2.3 From 187dd1005cd92c530146d7f5b0a89b368b09c3e9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 28 Jun 2017 15:24:53 +0200 Subject: Add specs for delayed stage_id background migrations --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index bfeb09f6da1..a73456af386 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,10 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .find_in_batches(batch_size: BATCH_SIZE) .with_index do |builds, batch| - migrations = builds.map { |build| [MIGRATION, [build.id]] } - - BackgroundMigrationWorker.perform_bulk(*migrations) + builds.each do |build| + schedule = (batch - 1) * 5.minutes + BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) + end end end -- cgit v1.2.3 From 69736f3927160bd362e165b4cd9e78912a3c30c0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:10:29 +0200 Subject: Use ActiveRecord 5 batching to schedule bg migration --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index a73456af386..9e95216b35a 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -13,10 +13,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration def up Build.where(stage_id: nil) - .find_in_batches(batch_size: BATCH_SIZE) - .with_index do |builds, batch| - builds.each do |build| - schedule = (batch - 1) * 5.minutes + .in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + relation.each do |build| BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) end end -- cgit v1.2.3 From 8bd9cb6c87a1a1c51830360fe1aa1a228f9c768e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:14:54 +0200 Subject: Perform stage_id ref backgound migration in bulks --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 9e95216b35a..c54e8bde095 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -15,9 +15,9 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil) .in_batches(of: BATCH_SIZE) do |relation, index| schedule = index * 5.minutes - relation.each do |build| - BackgroundMigrationWorker.perform_at(schedule, MIGRATION, [build.id]) - end + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.3 From 42556419c99522e921299a3e247f115f580be7f1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Jun 2017 12:26:37 +0200 Subject: Improve specs for background stage_id ref migration --- ...20170628080858_migrate_stage_id_reference_in_background.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index c54e8bde095..1d95fc62c87 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,13 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil) - .in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| + schedule = index * 5.minutes + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) - end + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + end end def down -- cgit v1.2.3 From 134f204ed8f5dd80e44463338ae93f3d905ca7af Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 13:03:47 +0200 Subject: Do not override original AR5 batching interface --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 1d95fc62c87..30849ea1361 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -12,9 +12,12 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration end def up - Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation, index| - schedule = index * 5.minutes + index = 1 + + Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end -- cgit v1.2.3 From 9c7c95c768cd5294dd085c2fc2425fae91c4c689 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Jun 2017 14:22:23 +0200 Subject: Add initial changes for stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb new file mode 100644 index 00000000000..b4b76893595 --- /dev/null +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -0,0 +1,76 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + + scope :relevant, -> do + where(status: %w[pending running success failed canceled skipped manual]) + end + + scope :created, -> { where(status: 'created') } + scope :running, -> { where(status: 'running') } + scope :pending, -> { where(status: 'pending') } + scope :success, -> { where(status: 'success') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } + + scope :failed_but_allowed, -> do + where(allow_failure: true, status: [:failed, :canceled]) + end + + scope :exclude_ignored, -> do + where("allow_failure = ? OR status IN (?)", + false, all_state_names - [:failed, :canceled, :manual]) + end + + def status_sql + scope_relevant = relevant.exclude_ignored + scope_warnings = relevant.failed_but_allowed + + builds = scope_relevant.select('count(*)').to_sql + created = scope_relevant.created.select('count(*)').to_sql + success = scope_relevant.success.select('count(*)').to_sql + manual = scope_relevant.manual.select('count(*)').to_sql + pending = scope_relevant.pending.select('count(*)').to_sql + running = scope_relevant.running.select('count(*)').to_sql + skipped = scope_relevant.skipped.select('count(*)').to_sql + canceled = scope_relevant.canceled.select('count(*)').to_sql + warnings = scope_warnings.select('count(*) > 0').to_sql + + "(CASE + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' + WHEN (#{builds})=(#{skipped}) THEN 'skipped' + WHEN (#{builds})=(#{success}) THEN 'success' + WHEN (#{builds})=(#{created}) THEN 'created' + WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' + WHEN (#{running})+(#{pending})>0 THEN 'running' + WHEN (#{manual})>0 THEN 'manual' + WHEN (#{created})>0 THEN 'running' + ELSE 'failed' + END)" + end + end + + def up + execute <<-SQL.strip_heredoc + SQL + end + + def down + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end + + private + +end -- cgit v1.2.3 From f6966d96ec5941db364a2c8d9d2d80d3aa7d20f2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Jul 2017 13:02:51 +0200 Subject: Reduce a delay between stage_id scheduled migrations --- .../20170628080858_migrate_stage_id_reference_in_background.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb index 30849ea1361..ebec4cb6bb7 100644 --- a/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb +++ b/db/post_migrate/20170628080858_migrate_stage_id_reference_in_background.rb @@ -16,7 +16,7 @@ class MigrateStageIdReferenceInBackground < ActiveRecord::Migration Build.where(stage_id: nil).in_batches(of: BATCH_SIZE) do |relation| jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes + schedule = index * 2.minutes index += 1 BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) -- cgit v1.2.3 From 7103c4a707157594c261ba2f68fbb649ca4df769 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 09:20:18 +0200 Subject: Extend stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 55 +++++++++++++--------- 1 file changed, 34 insertions(+), 21 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index b4b76893595..8c6de84adf5 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,13 +5,17 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! + STATUSES = { created: 0, pending: 1, running: 2, success: 3, + failed: 4, canceled: 5, skipped: 6, manual: 7 } + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + end + class Build < ActiveRecord::Base self.table_name = 'ci_builds' - scope :relevant, -> do - where(status: %w[pending running success failed canceled skipped manual]) - end - + scope :latest, -> { where(retried: [false, nil]) } scope :created, -> { where(status: 'created') } scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } @@ -27,12 +31,12 @@ class MigrateStagesStatuses < ActiveRecord::Migration scope :exclude_ignored, -> do where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled, :manual]) + false, %w[created pending running success skipped]) end - def status_sql - scope_relevant = relevant.exclude_ignored - scope_warnings = relevant.failed_but_allowed + def self.status_sql + scope_relevant = latest.exclude_ignored + scope_warnings = latest.failed_but_allowed builds = scope_relevant.select('count(*)').to_sql created = scope_relevant.created.select('count(*)').to_sql @@ -45,24 +49,33 @@ class MigrateStagesStatuses < ActiveRecord::Migration warnings = scope_warnings.select('count(*) > 0').to_sql "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN 'success' - WHEN (#{builds})=(#{skipped}) THEN 'skipped' - WHEN (#{builds})=(#{success}) THEN 'success' - WHEN (#{builds})=(#{created}) THEN 'created' - WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' - WHEN (#{running})+(#{pending})>0 THEN 'running' - WHEN (#{manual})>0 THEN 'manual' - WHEN (#{created})>0 THEN 'running' - ELSE 'failed' + WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} + WHEN (#{manual})>0 THEN #{STATUSES[:manual]} + WHEN (#{created})>0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} END)" end end def up - execute <<-SQL.strip_heredoc - SQL + Stage.all.in_batches(of: 10000) do |relation| + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = #{status_sql} + WHERE id = (#{relation.select(:id).to_sql}) + SQL + end end def down -- cgit v1.2.3 From d60ce6e9f44eba769a6ad595014ae96095169dd2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 13:43:26 +0200 Subject: Implement initial working stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 46 ++++++++++------------ 1 file changed, 20 insertions(+), 26 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 8c6de84adf5..62542ed0001 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -48,34 +48,31 @@ class MigrateStagesStatuses < ActiveRecord::Migration canceled = scope_relevant.canceled.select('count(*)').to_sql warnings = scope_warnings.select('count(*) > 0').to_sql - "(CASE - WHEN (#{builds})=(#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds})=(#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds})=(#{success})+(#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running})+(#{pending})>0 THEN '#{STATUSES[:running]} - WHEN (#{manual})>0 THEN #{STATUSES[:manual]} - WHEN (#{created})>0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END)" + <<-SQL.strip_heredoc + (CASE + WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} + WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} + WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} + WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} + WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} + WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} + WHEN (#{created}) > 0 THEN #{STATUSES[:running]} + ELSE #{STATUSES[:failed]} + END) + SQL end end def up - Stage.all.in_batches(of: 10000) do |relation| - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + status_sql = Build + .where('ci_builds.commit_id = ci_stages.pipeline_id') + .where('ci_builds.stage = ci_stages.name') + .status_sql - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = #{status_sql} - WHERE id = (#{relation.select(:id).to_sql}) - SQL - end + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) end def down @@ -83,7 +80,4 @@ class MigrateStagesStatuses < ActiveRecord::Migration UPDATE ci_stages SET status = null SQL end - - private - end -- cgit v1.2.3 From 93d217bda639b94c129afd71343e429f935a4ada Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Jul 2017 16:58:35 +0200 Subject: Migrate only old stages without status that is set --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 62542ed0001..9dac91960ff 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -72,7 +72,9 @@ class MigrateStagesStatuses < ActiveRecord::Migration .where('ci_builds.stage = ci_stages.name') .status_sql - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) + update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| + query.where(table[:status].eq(nil)) + end end def down -- cgit v1.2.3 From c5f1e1a70bd79b36fe8cfda75b7366dd8ee90d66 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 09:11:15 +0200 Subject: Disable statement timeout in stages statuses migration --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 9dac91960ff..c0a5294720d 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -8,10 +8,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration STATUSES = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 } - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end - class Build < ActiveRecord::Base self.table_name = 'ci_builds' @@ -67,6 +63,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def up + disable_statement_timeout + status_sql = Build .where('ci_builds.commit_id = ci_stages.pipeline_id') .where('ci_builds.stage = ci_stages.name') @@ -78,6 +76,8 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def down + disable_statement_timeout + execute <<-SQL.strip_heredoc UPDATE ci_stages SET status = null SQL -- cgit v1.2.3 From 6c477d5b9496829eb5cb56ef32a0dd813be7dc16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Jul 2017 10:54:48 +0200 Subject: Move stages status migration to the background worker --- .../20170630111158_migrate_stages_statuses.rb | 71 +++------------------- 1 file changed, 10 insertions(+), 61 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index c0a5294720d..2bc067e5d90 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -5,73 +5,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 } + BATCH_SIZE = 10000 + MIGRATION = 'MigrateStageStatus'.freeze - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' end def up - disable_statement_timeout + index = 1 - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql + Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| + jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } + schedule = index * 5.minutes + index += 1 - update_column_in_batches(:ci_stages, :status, Arel.sql("(#{status_sql})")) do |table, query| - query.where(table[:status].eq(nil)) + BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) end end -- cgit v1.2.3 From 7082530d555ad98fede2823d2123622abaf1c3a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 10 Jul 2017 15:42:19 +0200 Subject: Schedule stages statuses bg migrations in batches --- db/post_migrate/20170630111158_migrate_stages_statuses.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb index 2bc067e5d90..1641e550480 100644 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ b/db/post_migrate/20170630111158_migrate_stages_statuses.rb @@ -6,21 +6,22 @@ class MigrateStagesStatuses < ActiveRecord::Migration disable_ddl_transaction! BATCH_SIZE = 10000 + RANGE_SIZE = 1000 MIGRATION = 'MigrateStageStatus'.freeze class Stage < ActiveRecord::Base self.table_name = 'ci_stages' + include ::EachBatch end def up - index = 1 + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes - Stage.where(status: nil).in_batches(of: BATCH_SIZE) do |relation| - jobs = relation.pluck(:id).map { |id| [MIGRATION, [id]] } - schedule = index * 5.minutes - index += 1 - - BackgroundMigrationWorker.perform_bulk_in(schedule, jobs) + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end end end -- cgit v1.2.3 From bb67b4749b5b4c62d4235c90dc0320967f850cdd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 11 Jul 2017 14:31:04 +0200 Subject: Update version number of stages statuses migration --- .../20170630111158_migrate_stages_statuses.rb | 35 ---------------------- .../20170711145558_migrate_stages_statuses.rb | 35 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) delete mode 100644 db/post_migrate/20170630111158_migrate_stages_statuses.rb create mode 100644 db/post_migrate/20170711145558_migrate_stages_statuses.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170630111158_migrate_stages_statuses.rb b/db/post_migrate/20170630111158_migrate_stages_statuses.rb deleted file mode 100644 index 1641e550480..00000000000 --- a/db/post_migrate/20170630111158_migrate_stages_statuses.rb +++ /dev/null @@ -1,35 +0,0 @@ -class MigrateStagesStatuses < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - disable_ddl_transaction! - - BATCH_SIZE = 10000 - RANGE_SIZE = 1000 - MIGRATION = 'MigrateStageStatus'.freeze - - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - include ::EachBatch - end - - def up - Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| - relation.each_batch(of: RANGE_SIZE) do |batch| - range = relation.pluck('MIN(id)', 'MAX(id)').first - schedule = index * 5.minutes - - BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) - end - end - end - - def down - disable_statement_timeout - - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL - end -end diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb new file mode 100644 index 00000000000..1641e550480 --- /dev/null +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -0,0 +1,35 @@ +class MigrateStagesStatuses < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 10000 + RANGE_SIZE = 1000 + MIGRATION = 'MigrateStageStatus'.freeze + + class Stage < ActiveRecord::Base + self.table_name = 'ci_stages' + include ::EachBatch + end + + def up + Stage.where(status: nil).each_batch(of: BATCH_SIZE) do |relation, index| + relation.each_batch(of: RANGE_SIZE) do |batch| + range = relation.pluck('MIN(id)', 'MAX(id)').first + schedule = index * 5.minutes + + BackgroundMigrationWorker.perform_in(schedule, MIGRATION, range) + end + end + end + + def down + disable_statement_timeout + + execute <<-SQL.strip_heredoc + UPDATE ci_stages SET status = null + SQL + end +end -- cgit v1.2.3 From 13a15e7009e292919109ea911640a627dbd0e327 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 31 Jul 2017 12:23:04 +0200 Subject: Use update_column_in_batches helper in stages migration --- db/post_migrate/20170711145558_migrate_stages_statuses.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb index 1641e550480..5a24fb1307f 100644 --- a/db/post_migrate/20170711145558_migrate_stages_statuses.rb +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -28,8 +28,6 @@ class MigrateStagesStatuses < ActiveRecord::Migration def down disable_statement_timeout - execute <<-SQL.strip_heredoc - UPDATE ci_stages SET status = null - SQL + update_column_in_batches(:ci_stages, :status, nil) end end -- cgit v1.2.3