Merge branch 'all-ci-offline-migrations' into 'master'

All CI offline migrations

See merge request !9730

8 files changed, 272 insertions, 102 deletions

diff --git a/db/migrate/20170216135621_add_index_for_latest_successful_pipeline.rb b/db/migrate/20170216135621_add_index_for_latest_successful_pipeline.rb
index 65adc90c2c1..8a96a784c97 100644
--- a/db/migrate/20170216135621_add_index_for_latest_successful_pipeline.rb
+++ b/db/migrate/20170216135621_add_index_for_latest_successful_pipeline.rb
@@ -5,7 +5,7 @@ class AddIndexForLatestSuccessfulPipeline < ActiveRecord::Migration
   disable_ddl_transaction!
 
   def up
-    add_concurrent_index :ci_commits, [:gl_project_id, :ref, :status]
+    add_concurrent_index(:ci_commits, [:gl_project_id, :ref, :status])
   end
 
   def down
diff --git a/db/migrate/20170222143317_drop_ci_projects.rb b/db/migrate/20170222143317_drop_ci_projects.rb
new file mode 100644
index 00000000000..4db8658f36f
--- /dev/null
+++ b/db/migrate/20170222143317_drop_ci_projects.rb
@@ -0,0 +1,34 @@
+class DropCiProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    drop_table :ci_projects
+  end
+
+  def down
+    create_table "ci_projects", force: :cascade do |t|
+      t.string "name"
+      t.integer "timeout", default: 3600, null: false
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.string "token"
+      t.string "default_ref"
+      t.string "path"
+      t.boolean "always_build", default: false, null: false
+      t.integer "polling_interval"
+      t.boolean "public", default: false, null: false
+      t.string "ssh_url_to_repo"
+      t.integer "gitlab_id"
+      t.boolean "allow_git_fetch", default: true, null: false
+      t.string "email_recipients", default: "", null: false
+      t.boolean "email_add_pusher", default: true, null: false
+      t.boolean "email_only_broken_builds", default: true, null: false
+      t.string "skip_refs"
+      t.string "coverage_regex"
+      t.boolean "shared_runners_enabled", default: false
+      t.text "generated_yaml_config"
+    end
+  end
+end
diff --git a/db/migrate/20170222143500_remove_old_project_id_columns.rb b/db/migrate/20170222143500_remove_old_project_id_columns.rb
new file mode 100644
index 00000000000..eac93e8e407
--- /dev/null
+++ b/db/migrate/20170222143500_remove_old_project_id_columns.rb
@@ -0,0 +1,28 @@
+class RemoveOldProjectIdColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Unused columns are being removed.'
+
+  def up
+    remove_index :ci_builds, :project_id if
+      index_exists?(:ci_builds, :project_id)
+
+    remove_column :ci_builds, :project_id
+    remove_column :ci_commits, :project_id
+    remove_column :ci_runner_projects, :project_id
+    remove_column :ci_triggers, :project_id
+    remove_column :ci_variables, :project_id
+  end
+
+  def down
+    add_column :ci_builds, :project_id, :integer
+    add_column :ci_commits, :project_id, :integer
+    add_column :ci_runner_projects, :project_id, :integer
+    add_column :ci_triggers, :project_id, :integer
+    add_column :ci_variables, :project_id, :integer
+
+    add_concurrent_index :ci_builds, :project_id
+  end
+end
diff --git a/db/migrate/20170222143603_rename_gl_project_id_to_project_id.rb b/db/migrate/20170222143603_rename_gl_project_id_to_project_id.rb
new file mode 100644
index 00000000000..7c19d471557
--- /dev/null
+++ b/db/migrate/20170222143603_rename_gl_project_id_to_project_id.rb
@@ -0,0 +1,14 @@
+class RenameGlProjectIdToProjectId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Renaming an actively used column.'
+
+  def change
+    rename_column :ci_builds, :gl_project_id, :project_id
+    rename_column :ci_commits, :gl_project_id, :project_id
+    rename_column :ci_runner_projects, :gl_project_id, :project_id
+    rename_column :ci_triggers, :gl_project_id, :project_id
+    rename_column :ci_variables, :gl_project_id, :project_id
+  end
+end
diff --git a/db/migrate/20170301195939_rename_ci_commits_to_ci_pipelines.rb b/db/migrate/20170301195939_rename_ci_commits_to_ci_pipelines.rb
new file mode 100644
index 00000000000..4f061d96392
--- /dev/null
+++ b/db/migrate/20170301195939_rename_ci_commits_to_ci_pipelines.rb
@@ -0,0 +1,10 @@
+class RenameCiCommitsToCiPipelines < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Rename table ci_commits to ci_pipelines'
+
+  def change
+    rename_table 'ci_commits', 'ci_pipelines'
+  end
+end
diff --git a/db/migrate/20170301205639_remove_unused_ci_tables_and_columns.rb b/db/migrate/20170301205639_remove_unused_ci_tables_and_columns.rb
new file mode 100644
index 00000000000..1e2abea5254
--- /dev/null
+++ b/db/migrate/20170301205639_remove_unused_ci_tables_and_columns.rb
@@ -0,0 +1,83 @@
+class RemoveUnusedCiTablesAndColumns < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON =
+    'Remove unused columns in used tables.' \
+    ' Downtime required in case Rails caches them'
+
+  def up
+    %w[ci_application_settings
+       ci_events
+       ci_jobs
+       ci_sessions
+       ci_taggings
+       ci_tags].each do |table|
+      drop_table(table)
+    end
+
+    remove_column :ci_pipelines, :push_data, :text
+    remove_column :ci_builds, :job_id, :integer
+    remove_column :ci_builds, :deploy, :boolean
+  end
+
+  def down
+    add_column :ci_builds, :deploy, :boolean
+    add_column :ci_builds, :job_id, :integer
+    add_column :ci_pipelines, :push_data, :text
+
+    create_table "ci_tags", force: :cascade do |t|
+      t.string "name"
+      t.integer "taggings_count", default: 0
+    end
+
+    create_table "ci_taggings", force: :cascade do |t|
+      t.integer "tag_id"
+      t.integer "taggable_id"
+      t.string "taggable_type"
+      t.integer "tagger_id"
+      t.string "tagger_type"
+      t.string "context", limit: 128
+      t.datetime "created_at"
+    end
+
+    add_index "ci_taggings", %w[taggable_id taggable_type context]
+
+    create_table "ci_sessions", force: :cascade do |t|
+      t.string "session_id", null: false
+      t.text "data"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+
+    create_table "ci_jobs", force: :cascade do |t|
+      t.integer "project_id", null: false
+      t.text "commands"
+      t.boolean "active", default: true, null: false
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.string "name"
+      t.boolean "build_branches", default: true, null: false
+      t.boolean "build_tags", default: false, null: false
+      t.string "job_type", default: "parallel"
+      t.string "refs"
+      t.datetime "deleted_at"
+    end
+
+    create_table "ci_events", force: :cascade do |t|
+      t.integer "project_id"
+      t.integer "user_id"
+      t.integer "is_admin"
+      t.text "description"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+
+    create_table "ci_application_settings", force: :cascade do |t|
+      t.boolean "all_broken_builds"
+      t.boolean "add_pusher"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+  end
+end
diff --git a/db/post_migrate/20170301205640_migrate_build_events_to_pipeline_events.rb b/db/post_migrate/20170301205640_migrate_build_events_to_pipeline_events.rb
new file mode 100644
index 00000000000..2dd14ee5a78
--- /dev/null
+++ b/db/post_migrate/20170301205640_migrate_build_events_to_pipeline_events.rb
@@ -0,0 +1,87 @@
+class MigrateBuildEventsToPipelineEvents < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  include Gitlab::Database
+
+  DOWNTIME = false
+
+  def up
+    Gitlab::Database.with_connection_pool(2) do |pool|
+      threads = []
+
+      threads << Thread.new do
+        pool.with_connection do |connection|
+          Thread.current[:foreign_key_connection] = connection
+
+          execute(<<-SQL.strip_heredoc)
+            UPDATE services
+              SET properties = replace(properties,
+                                       'notify_only_broken_builds',
+                                       'notify_only_broken_pipelines')
+                , pipeline_events = #{true_value}
+                , build_events = #{false_value}
+            WHERE type IN
+              ('SlackService', 'MattermostService', 'HipchatService')
+              AND build_events = #{true_value};
+          SQL
+        end
+      end
+
+      threads << Thread.new do
+        pool.with_connection do |connection|
+          Thread.current[:foreign_key_connection] = connection
+
+          execute(update_pipeline_services_sql)
+        end
+      end
+
+      threads.each(&:join)
+    end
+  end
+
+  def down
+    # Don't bother to migrate the data back
+  end
+
+  def connection
+    # Rails memoizes connection objects, but this causes them to be shared
+    # amongst threads; we don't want that.
+    Thread.current[:foreign_key_connection] || ActiveRecord::Base.connection
+  end
+
+  private
+
+  def update_pipeline_services_sql
+    if Gitlab::Database.postgresql?
+      <<-SQL
+        UPDATE services
+          SET type = 'PipelinesEmailService'
+            , properties = replace(properties,
+                                   'notify_only_broken_builds',
+                                   'notify_only_broken_pipelines')
+            , pipeline_events = #{true_value}
+            , build_events = #{false_value}
+        WHERE type = 'BuildsEmailService'
+        AND
+          (SELECT 1 FROM services pipeline_services
+             WHERE pipeline_services.project_id = services.project_id
+               AND pipeline_services.type = 'PipelinesEmailService' LIMIT 1)
+          IS NULL;
+      SQL
+    else
+      <<-SQL
+        UPDATE services build_services
+         LEFT OUTER JOIN services pipeline_services
+           ON build_services.project_id = pipeline_services.project_id
+          AND pipeline_services.type = 'PipelinesEmailService'
+          SET build_services.type = 'PipelinesEmailService'
+            , build_services.properties = replace(build_services.properties,
+                                         'notify_only_broken_builds',
+                                         'notify_only_broken_pipelines')
+            , build_services.pipeline_events = #{true_value}
+            , build_services.build_events = #{false_value}
+        WHERE build_services.type = 'BuildsEmailService'
+          AND pipeline_services.id IS NULL;
+      SQL
+    end.strip_heredoc
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3bef910c1d6..6ea2f088353 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -185,15 +185,7 @@ ActiveRecord::Schema.define(version: 20170315174634) do
 
   add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
 
-  create_table "ci_application_settings", force: :cascade do |t|
-    t.boolean "all_broken_builds"
-    t.boolean "add_pusher"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
   create_table "ci_builds", force: :cascade do |t|
-    t.integer "project_id"
     t.string "status"
     t.datetime "finished_at"
     t.text "trace"
@@ -204,9 +196,7 @@ ActiveRecord::Schema.define(version: 20170315174634) do
     t.float "coverage"
     t.integer "commit_id"
     t.text "commands"
-    t.integer "job_id"
     t.string "name"
-    t.boolean "deploy", default: false
     t.text "options"
     t.boolean "allow_failure", default: false, null: false
     t.string "stage"
@@ -219,7 +209,7 @@ ActiveRecord::Schema.define(version: 20170315174634) do
     t.string "target_url"
     t.string "description"
     t.text "artifacts_file"
-    t.integer "gl_project_id"
+    t.integer "project_id"
     t.text "artifacts_metadata"
     t.integer "erased_by_id"
     t.datetime "erased_at"
@@ -238,25 +228,22 @@ ActiveRecord::Schema.define(version: 20170315174634) do
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
   add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
-  add_index "ci_builds", ["gl_project_id"], name: "index_ci_builds_on_gl_project_id", using: :btree
   add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
   add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree
   add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree
   add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree
 
-  create_table "ci_commits", force: :cascade do |t|
-    t.integer "project_id"
+  create_table "ci_pipelines", force: :cascade do |t|
     t.string "ref"
     t.string "sha"
     t.string "before_sha"
-    t.text "push_data"
     t.datetime "created_at"
     t.datetime "updated_at"
     t.boolean "tag", default: false
     t.text "yaml_errors"
     t.datetime "committed_at"
-    t.integer "gl_project_id"
+    t.integer "project_id"
     t.string "status"
     t.datetime "started_at"
     t.datetime "finished_at"
@@ -265,67 +252,20 @@ ActiveRecord::Schema.define(version: 20170315174634) do
     t.integer "lock_version"
   end
 
-  add_index "ci_commits", ["gl_project_id", "ref", "status"], name: "index_ci_commits_on_gl_project_id_and_ref_and_status", using: :btree
-  add_index "ci_commits", ["gl_project_id", "sha"], name: "index_ci_commits_on_gl_project_id_and_sha", using: :btree
-  add_index "ci_commits", ["gl_project_id"], name: "index_ci_commits_on_gl_project_id", using: :btree
-  add_index "ci_commits", ["status"], name: "index_ci_commits_on_status", using: :btree
-  add_index "ci_commits", ["user_id"], name: "index_ci_commits_on_user_id", using: :btree
-
-  create_table "ci_events", force: :cascade do |t|
-    t.integer "project_id"
-    t.integer "user_id"
-    t.integer "is_admin"
-    t.text "description"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
-  create_table "ci_jobs", force: :cascade do |t|
-    t.integer "project_id", null: false
-    t.text "commands"
-    t.boolean "active", default: true, null: false
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string "name"
-    t.boolean "build_branches", default: true, null: false
-    t.boolean "build_tags", default: false, null: false
-    t.string "job_type", default: "parallel"
-    t.string "refs"
-    t.datetime "deleted_at"
-  end
-
-  create_table "ci_projects", force: :cascade do |t|
-    t.string "name"
-    t.integer "timeout", default: 3600, null: false
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string "token"
-    t.string "default_ref"
-    t.string "path"
-    t.boolean "always_build", default: false, null: false
-    t.integer "polling_interval"
-    t.boolean "public", default: false, null: false
-    t.string "ssh_url_to_repo"
-    t.integer "gitlab_id"
-    t.boolean "allow_git_fetch", default: true, null: false
-    t.string "email_recipients", default: "", null: false
-    t.boolean "email_add_pusher", default: true, null: false
-    t.boolean "email_only_broken_builds", default: true, null: false
-    t.string "skip_refs"
-    t.string "coverage_regex"
-    t.boolean "shared_runners_enabled", default: false
-    t.text "generated_yaml_config"
-  end
+  add_index "ci_pipelines", ["project_id", "ref", "status"], name: "index_ci_pipelines_on_project_id_and_ref_and_status", using: :btree
+  add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
+  add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
+  add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
+  add_index "ci_pipelines", ["user_id"], name: "index_ci_pipelines_on_user_id", using: :btree
 
   create_table "ci_runner_projects", force: :cascade do |t|
     t.integer "runner_id", null: false
-    t.integer "project_id"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.integer "gl_project_id"
+    t.integer "project_id"
   end
 
-  add_index "ci_runner_projects", ["gl_project_id"], name: "index_ci_runner_projects_on_gl_project_id", using: :btree
+  add_index "ci_runner_projects", ["project_id"], name: "index_ci_runner_projects_on_project_id", using: :btree
   add_index "ci_runner_projects", ["runner_id"], name: "index_ci_runner_projects_on_runner_id", using: :btree
 
   create_table "ci_runners", force: :cascade do |t|
@@ -349,30 +289,6 @@ ActiveRecord::Schema.define(version: 20170315174634) do
   add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree
   add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree
 
-  create_table "ci_sessions", force: :cascade do |t|
-    t.string "session_id", null: false
-    t.text "data"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-  end
-
-  create_table "ci_taggings", force: :cascade do |t|
-    t.integer "tag_id"
-    t.integer "taggable_id"
-    t.string "taggable_type"
-    t.integer "tagger_id"
-    t.string "tagger_type"
-    t.string "context", limit: 128
-    t.datetime "created_at"
-  end
-
-  add_index "ci_taggings", ["taggable_id", "taggable_type", "context"], name: "index_ci_taggings_on_taggable_id_and_taggable_type_and_context", using: :btree
-
-  create_table "ci_tags", force: :cascade do |t|
-    t.string "name"
-    t.integer "taggings_count", default: 0
-  end
-
   create_table "ci_trigger_requests", force: :cascade do |t|
     t.integer "trigger_id", null: false
     t.text "variables"
@@ -385,28 +301,26 @@ ActiveRecord::Schema.define(version: 20170315174634) do
 
   create_table "ci_triggers", force: :cascade do |t|
     t.string "token"
-    t.integer "project_id"
     t.datetime "deleted_at"
     t.datetime "created_at"
     t.datetime "updated_at"
-    t.integer "gl_project_id"
+    t.integer "project_id"
     t.integer "owner_id"
     t.string "description"
   end
 
-  add_index "ci_triggers", ["gl_project_id"], name: "index_ci_triggers_on_gl_project_id", using: :btree
+  add_index "ci_triggers", ["project_id"], name: "index_ci_triggers_on_project_id", using: :btree
 
   create_table "ci_variables", force: :cascade do |t|
-    t.integer "project_id"
     t.string "key"
     t.text "value"
     t.text "encrypted_value"
     t.string "encrypted_value_salt"
     t.string "encrypted_value_iv"
-    t.integer "gl_project_id"
+    t.integer "project_id"
   end
 
-  add_index "ci_variables", ["gl_project_id"], name: "index_ci_variables_on_gl_project_id", using: :btree
+  add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree
 
   create_table "deploy_keys_projects", force: :cascade do |t|
     t.integer "deploy_key_id", null: false
@@ -1377,7 +1291,7 @@ ActiveRecord::Schema.define(version: 20170315174634) do
   add_foreign_key "labels", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "lists", "boards"
   add_foreign_key "lists", "labels"
-  add_foreign_key "merge_request_metrics", "ci_commits", column: "pipeline_id", on_delete: :cascade
+  add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "issues", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade