Merge branch 'master' into per-project-pipeline-iid

6 files changed, 102 insertions, 5 deletions

diff --git a/db/migrate/20140313092127_init_schema.rb b/db/migrate/20140313092127_init_schema.rb
index e5c3b65fa8d..895298ed4ed 100644
--- a/db/migrate/20140313092127_init_schema.rb
+++ b/db/migrate/20140313092127_init_schema.rb
@@ -1,4 +1,6 @@
 class InitSchema < ActiveRecord::Migration
+  DOWNTIME = true
+
   def up
     create_table "broadcast_messages", force: :cascade do |t|
       t.text "message", null: false
diff --git a/db/migrate/20180420010616_cleanup_build_stage_migration.rb b/db/migrate/20180420010616_cleanup_build_stage_migration.rb
index 0342695ec3d..24777294101 100644
--- a/db/migrate/20180420010616_cleanup_build_stage_migration.rb
+++ b/db/migrate/20180420010616_cleanup_build_stage_migration.rb
@@ -2,6 +2,7 @@ class CleanupBuildStageMigration < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
   DOWNTIME = false
+  TMP_INDEX = 'tmp_id_stage_partial_null_index'.freeze
 
   disable_ddl_transaction!
 
@@ -13,16 +14,48 @@ class CleanupBuildStageMigration < ActiveRecord::Migration
   end
 
   def up
+    disable_statement_timeout
+
+    ##
+    # We steal from the background migrations queue to catch up with the
+    # scheduled migrations set.
+    #
     Gitlab::BackgroundMigration.steal('MigrateBuildStage')
 
+    ##
+    # We add temporary index, to make iteration over batches more performant.
+    # Conditional here is to avoid the need of doing that in a separate
+    # migration file to make this operation idempotent.
+    #
+    unless index_exists_by_name?(:ci_builds, TMP_INDEX)
+      add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', name: TMP_INDEX)
+    end
+
+    ##
+    # We check if there are remaining rows that should be migrated (for example
+    # if Sidekiq / Redis fails / is restarted, what could result in not all
+    # background migrations being executed correctly.
+    #
+    # We migrate remaining rows synchronously in a blocking way, to make sure
+    # that when this migration is done we are confident that all rows are
+    # already migrated.
+    #
     Build.where('stage_id IS NULL').each_batch(of: 50) do |batch|
       range = batch.pluck('MIN(id)', 'MAX(id)').first
 
       Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range)
     end
+
+    ##
+    # We remove temporary index, because it is not required during standard
+    # operations and runtime.
+    #
+    remove_concurrent_index_by_name(:ci_builds, TMP_INDEX)
   end
 
   def down
-    # noop
+    if index_exists_by_name?(:ci_builds, TMP_INDEX)
+      remove_concurrent_index_by_name(:ci_builds, TMP_INDEX)
+    end
   end
 end
diff --git a/db/migrate/20180508100222_add_not_null_constraint_to_project_mirror_data_foreign_key.rb b/db/migrate/20180508100222_add_not_null_constraint_to_project_mirror_data_foreign_key.rb
new file mode 100644
index 00000000000..82087d15ccb
--- /dev/null
+++ b/db/migrate/20180508100222_add_not_null_constraint_to_project_mirror_data_foreign_key.rb
@@ -0,0 +1,21 @@
+class AddNotNullConstraintToProjectMirrorDataForeignKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  class ProjectImportState < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_mirror_data'
+  end
+
+  def up
+    ProjectImportState.where(project_id: nil).delete_all
+
+    change_column_null :project_mirror_data, :project_id, false
+  end
+
+  def down
+    change_column_null :project_mirror_data, :project_id, true
+  end
+end
diff --git a/db/migrate/20180508102840_add_unique_constraint_to_project_mirror_data_project_id_index.rb b/db/migrate/20180508102840_add_unique_constraint_to_project_mirror_data_project_id_index.rb
new file mode 100644
index 00000000000..acb976b52fa
--- /dev/null
+++ b/db/migrate/20180508102840_add_unique_constraint_to_project_mirror_data_project_id_index.rb
@@ -0,0 +1,31 @@
+class AddUniqueConstraintToProjectMirrorDataProjectIdIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:project_mirror_data,
+                         :project_id,
+                         unique: true,
+                         name: 'index_project_mirror_data_on_project_id_unique')
+
+    remove_concurrent_index_by_name(:project_mirror_data, 'index_project_mirror_data_on_project_id')
+
+    rename_index(:project_mirror_data,
+                 'index_project_mirror_data_on_project_id_unique',
+                 'index_project_mirror_data_on_project_id')
+  end
+
+  def down
+    rename_index(:project_mirror_data,
+                 'index_project_mirror_data_on_project_id',
+                 'index_project_mirror_data_on_project_id_old')
+
+    add_concurrent_index(:project_mirror_data, :project_id)
+
+    remove_concurrent_index_by_name(:project_mirror_data,
+                                    'index_project_mirror_data_on_project_id_old')
+  end
+end
diff --git a/db/migrate/20180508135515_set_runner_type_not_null.rb b/db/migrate/20180508135515_set_runner_type_not_null.rb
new file mode 100644
index 00000000000..dd043ec7179
--- /dev/null
+++ b/db/migrate/20180508135515_set_runner_type_not_null.rb
@@ -0,0 +1,9 @@
+class SetRunnerTypeNotNull < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    change_column_null(:ci_runners, :runner_type, false)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2cf6ff3da9f..77492236f72 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180508055821) do
+ActiveRecord::Schema.define(version: 20180508135515) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -499,7 +499,7 @@ ActiveRecord::Schema.define(version: 20180508055821) do
     t.integer "access_level", default: 0, null: false
     t.string "ip_address"
     t.integer "maximum_timeout"
-    t.integer "runner_type", limit: 2
+    t.integer "runner_type", limit: 2, null: false
   end
 
   add_index "ci_runners", ["contacted_at"], name: "index_ci_runners_on_contacted_at", using: :btree
@@ -1531,14 +1531,14 @@ ActiveRecord::Schema.define(version: 20180508055821) do
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
   create_table "project_mirror_data", force: :cascade do |t|
-    t.integer "project_id"
+    t.integer "project_id", null: false
     t.string "status"
     t.string "jid"
     t.text "last_error"
   end
 
   add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
-  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree
+  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", unique: true, using: :btree
   add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
 
   create_table "project_statistics", force: :cascade do |t|
@@ -2150,6 +2150,7 @@ ActiveRecord::Schema.define(version: 20180508055821) do
   add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade
   add_foreign_key "ci_build_trace_sections", "projects", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
+  add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
   add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
   add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade