Add table for files in merge request diffs

This adds an ID-less table containing one row per file, per merge request
diff. It has a column for each attribute on Gitlab::Git::Diff that is serialised
currently, with the advantage that we can easily query the attributes of this
new table.

It does not migrate existing data, so we have fallback code when the legacy
st_diffs column is present instead. For a merge request diff to be valid, it
should have at most one of:

* Rows in this new table, with the correct merge_request_diff_id.
* A non-NULL st_diffs column.

It may have neither, if the diff is empty.

3 files changed, 14 insertions, 2 deletions

diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index ff2b1d08c3c..72183e8aad4 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -26,7 +26,8 @@ project_tree:
     - notes:
       - :author
       - :events
-    - :merge_request_diff
+    - merge_request_diff:
+      - :merge_request_diff_files
     - :events
     - :timelogs
     - label_links:
@@ -92,6 +93,8 @@ excluded_attributes:
     - :expired_at
   merge_request_diff:
     - :st_diffs
+  merge_request_diff_files:
+    - :diff
   issues:
     - :milestone_id
   merge_requests:
@@ -113,6 +116,8 @@ methods:
     - :type
   merge_request_diff:
     - :utf8_st_diffs
+  merge_request_diff_files:
+    - :utf8_diff
   merge_requests:
     - :diff_head_sha
   project:
diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb
index 48c09dafcb6..b48f63bcd7e 100644
--- a/lib/gitlab/import_export/json_hash_builder.rb
+++ b/lib/gitlab/import_export/json_hash_builder.rb
@@ -83,7 +83,9 @@ module Gitlab
       # +value+ existing model to be included in the hash
       # +json_config_hash+ the original hash containing the root model
       def add_model_value(current_key, value, json_config_hash)
-        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+        @attributes_finder.parse(value) do |hash|
+          value = { value => hash } unless value.is_a?(Hash)
+        end
 
         add_to_array(current_key, json_config_hash, value)
       end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 695852526cb..20580459046 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -71,6 +71,7 @@ module Gitlab
 
         @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
         set_st_diff_commits if @relation_name == :merge_request_diff
+        set_diff if @relation_name == :merge_request_diff_files
       end
 
       def update_user_references
@@ -202,6 +203,10 @@ module Gitlab
         HashUtil.deep_symbolize_array_with_date!(@relation_hash['st_commits'])
       end
 
+      def set_diff
+        @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
+      end
+
       def existing_or_new_object
         # Only find existing records to avoid mapping tables such as milestones
         # Otherwise always create the record, skipping the extra SELECT clause.