Fix saving diffs that are not valid UTF-8

Previously, we used Psych, which would:

1. Check if a string was encoded as binary, and not ASCII-compatible.
2. Add the !binary tag in that case.
3. Convert to base64.

We need to do the same thing, using a new column in place of the tag.

2 files changed, 23 insertions, 4 deletions

diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index 4b141945ab4..ec87aee9310 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -236,10 +236,21 @@ class MergeRequestDiff < ActiveRecord::Base
 
   def create_merge_request_diff_files(diffs)
     rows = diffs.map.with_index do |diff, index|
-      diff.to_hash.merge(
+      diff_hash = diff.to_hash.merge(
+        binary: false,
         merge_request_diff_id: self.id,
         relative_order: index
       )
+
+      # Compatibility with old diffs created with Psych.
+      diff_hash.tap do |hash|
+        diff_text = hash[:diff]
+
+        if diff_text.encoding == Encoding::BINARY && !diff_text.ascii_only?
+          hash[:binary] = true
+          hash[:diff] = [diff_text].pack('m0')
+        end
+      end
     end
 
     Gitlab::Database.bulk_insert('merge_request_diff_files', rows)
@@ -268,9 +279,7 @@ class MergeRequestDiff < ActiveRecord::Base
           st_diffs
         end
       elsif merge_request_diff_files.present?
-        merge_request_diff_files
-          .as_json(only: Gitlab::Git::Diff::SERIALIZE_KEYS)
-          .map(&:with_indifferent_access)
+        merge_request_diff_files.map(&:to_hash)
       end
   end
 
diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb
index 598ebd4d829..1199ff5af22 100644
--- a/app/models/merge_request_diff_file.rb
+++ b/app/models/merge_request_diff_file.rb
@@ -8,4 +8,14 @@ class MergeRequestDiffFile < ActiveRecord::Base
 
     encode_utf8(diff) if diff.respond_to?(:encoding)
   end
+
+  def diff
+    binary? ? super.unpack('m0').first : super
+  end
+
+  def to_hash
+    keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff]
+
+    as_json(only: keys).merge(diff: diff).with_indifferent_access
+  end
 end