Add config_options|variables to BuildMetadata

These are data columns that store runtime configuration
of build needed to execute it on runner and within pipeline.

The definition of this data is that once used, and when no longer
needed (due to retry capability) they can be freely removed.

They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking
support for json datatype on old enough version).

7 files changed, 47 insertions, 8 deletions

diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index d587c896712..7fcabc035ac 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -15,7 +15,6 @@ module Gitlab
           def from_commands(job)
             self.new(:script).tap do |step|
               step.script = job.options[:before_script].to_a + job.options[:script].to_a
-              step.script = job.commands.split("\n") if step.script.empty?
               step.timeout = job.metadata_timeout
               step.when = WHEN_ON_SUCCESS
             end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 50942fbdb40..3239743bfff 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -95,7 +95,7 @@ module Gitlab
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage, :retry,
+                  :artifacts, :environment, :coverage, :retry,
                   :parallel
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
@@ -121,10 +121,6 @@ module Gitlab
             @config.merge(to_hash.compact)
           end
 
-          def commands
-            (before_script_value.to_a + script_value.to_a).join("\n")
-          end
-
           def manual_action?
             self.when == 'manual'
           end
@@ -156,7 +152,6 @@ module Gitlab
             { name: name,
               before_script: before_script_value,
               script: script_value,
-              commands: commands,
               image: image_value,
               services: services_value,
               stage: stage_value,
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 172926b8ab0..15097188061 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -33,7 +33,6 @@ module Gitlab
 
         { stage_idx: @stages.index(job[:stage]),
           stage: job[:stage],
-          commands: job[:commands],
           tag_list: job[:tags] || [],
           name: job[:name].to_s,
           allow_failure: job[:ignore],
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 9fb1ae9f64b..a1a374cef4a 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -148,6 +148,7 @@ excluded_attributes:
     - :when
     - :artifacts_file
     - :artifacts_metadata
+    - :commands
   push_event_payload:
     - :event_id
   project_badges:
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index a0f4dcfb772..bce12103cce 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -150,6 +150,7 @@ module Gitlab
         if BUILD_MODELS.include?(@relation_name)
           @relation_hash.delete('trace') # old export files have trace
           @relation_hash.delete('token')
+          @relation_hash.delete('commands')
 
           imported_object
         elsif @relation_name == :merge_requests
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index a81cee0d6d2..99fa65e0e90 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -115,5 +115,15 @@ module Gitlab
 
       string_or_array.split(',').map(&:strip)
     end
+
+    def deep_indifferent_access(data)
+      if data.is_a?(Array)
+        data.map(&method(:deep_indifferent_access))
+      elsif data.is_a?(Hash)
+        data.with_indifferent_access
+      else
+        data
+      end
+    end
   end
 end
diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb
new file mode 100644
index 00000000000..93cb192087a
--- /dev/null
+++ b/lib/serializers/json.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Serializers
+  # This serializer exports data as JSON,
+  # it is designed to be used with interwork compatibility between MySQL and PostgreSQL
+  # implementations, as used version of MySQL does not support native json type
+  #
+  # Secondly, the loader makes the resulting hash to have deep indifferent access
+  class JSON
+    class << self
+      def dump(obj)
+        # MySQL stores data as text
+        # look at ./config/initializers/ar_mysql_jsonb_support.rb
+        if Gitlab::Database.mysql?
+          obj = ActiveSupport::JSON.encode(obj)
+        end
+
+        obj
+      end
+
+      def load(data)
+        return if data.nil?
+
+        # On MySQL we store data as text
+        # look at ./config/initializers/ar_mysql_jsonb_support.rb
+        if Gitlab::Database.mysql?
+          data = ActiveSupport::JSON.decode(data)
+        end
+
+        Gitlab::Utils.deep_indifferent_access(data)
+      end
+    end
+  end
+end