Remove serialised diff and commit columns

The st_commits and st_diffs columns on merge_request_diffs historically held the
YAML-serialised data for a merge request diff, in a variety of formats.

Since 9.5, these have been migrated in the background to two new tables:
merge_request_diff_commits and merge_request_diff_files. That has the advantage
that we can actually query the data (for instance, to find out how many commits
we've stored), and that it can't be in a variety of formats, but must match the
new schema.

This is the final step of that journey, where we drop those columns and remove
all references to them. This is a breaking change to the importer, because we
can no longer import diffs created in the old format, and we cannot guarantee
the export will be in the new format unless it was generated after this commit.

4 files changed, 59 insertions, 4 deletions

diff --git a/db/migrate/20171121135738_clean_up_from_merge_request_diffs_and_commits.rb b/db/migrate/20171121135738_clean_up_from_merge_request_diffs_and_commits.rb
new file mode 100644
index 00000000000..30cf08b29fc
--- /dev/null
+++ b/db/migrate/20171121135738_clean_up_from_merge_request_diffs_and_commits.rb
@@ -0,0 +1,36 @@
+class CleanUpFromMergeRequestDiffsAndCommits < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+  end
+
+  disable_ddl_transaction!
+
+  def up
+    Gitlab::BackgroundMigration.steal('DeserializeMergeRequestDiffsAndCommits')
+
+    # The literal '--- []\n' value is created by the import process and treated
+    # as null by the application, so we can ignore those - even if we were
+    # migrating, it wouldn't create any rows.
+    literal_prefix = Gitlab::Database.postgresql? ? 'E' : ''
+    non_empty = "
+      (st_commits IS NOT NULL AND st_commits != #{literal_prefix}'--- []\n')
+      OR
+      (st_diffs IS NOT NULL AND st_diffs != #{literal_prefix}'--- []\n')
+    ".squish
+
+    MergeRequestDiff.where(non_empty).each_batch(of: 500) do |relation, index|
+      range = relation.pluck('MIN(id)', 'MAX(id)').first
+
+      Gitlab::BackgroundMigration::DeserializeMergeRequestDiffsAndCommits.new.perform(*range)
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/limits_to_mysql.rb b/db/migrate/limits_to_mysql.rb
index 5cd9f3198e3..8f8d8f27410 100644
--- a/db/migrate/limits_to_mysql.rb
+++ b/db/migrate/limits_to_mysql.rb
@@ -3,8 +3,19 @@ class LimitsToMysql < ActiveRecord::Migration
   def up
     return unless ActiveRecord::Base.configurations[Rails.env]['adapter'] =~ /^mysql/
 
-    change_column :merge_request_diffs, :st_commits, :text, limit: 2147483647
-    change_column :merge_request_diffs, :st_diffs, :text, limit: 2147483647
+    # These columns were removed in 10.3, but this is called from two places:
+    # 1. A migration run after they were added, but before they were removed.
+    # 2. A rake task which can be run at any time.
+    #
+    # Because of item 2, we need these checks.
+    if column_exists?(:merge_request_diffs, :st_commits)
+      change_column :merge_request_diffs, :st_commits, :text, limit: 2147483647
+    end
+
+    if column_exists?(:merge_request_diffs, :st_diffs)
+      change_column :merge_request_diffs, :st_diffs, :text, limit: 2147483647
+    end
+
     change_column :snippets, :content, :text, limit: 2147483647
     change_column :notes, :st_diff, :text, limit: 2147483647
   end
diff --git a/db/post_migrate/20171121160421_remove_merge_request_diff_st_commits_and_st_diffs.rb b/db/post_migrate/20171121160421_remove_merge_request_diff_st_commits_and_st_diffs.rb
new file mode 100644
index 00000000000..3a7b2a7fac0
--- /dev/null
+++ b/db/post_migrate/20171121160421_remove_merge_request_diff_st_commits_and_st_diffs.rb
@@ -0,0 +1,10 @@
+class RemoveMergeRequestDiffStCommitsAndStDiffs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    remove_column :merge_request_diffs, :st_commits, :text
+    remove_column :merge_request_diffs, :st_diffs, :text
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 804bc8d6e37..292163ee9bf 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1011,8 +1011,6 @@ ActiveRecord::Schema.define(version: 20171124132536) do
 
   create_table "merge_request_diffs", force: :cascade do |t|
     t.string "state"
-    t.text "st_commits"
-    t.text "st_diffs"
     t.integer "merge_request_id", null: false
     t.datetime "created_at"
     t.datetime "updated_at"