Add latest changes from gitlab-org/gitlab@master

2 files changed, 116 insertions, 17 deletions

diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb
index 3aa2b16bc73..d207c215618 100644
--- a/app/services/ci/create_job_artifacts_service.rb
+++ b/app/services/ci/create_job_artifacts_service.rb
@@ -10,10 +10,24 @@ module Ci
     ].freeze
 
     def execute(job, artifacts_file, params, metadata_file: nil)
+      return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
+
+      artifact, artifact_metadata = build_artifact(job, artifacts_file, params, metadata_file)
+      result = parse_artifact(job, artifact)
+
+      return result unless result[:status] == :success
+
+      persist_artifact(job, artifact, artifact_metadata)
+    end
+
+    private
+
+    def build_artifact(job, artifacts_file, params, metadata_file)
       expire_in = params['expire_in'] ||
         Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-      job.job_artifacts.build(
+      artifact = Ci::JobArtifact.new(
+        job_id: job.id,
         project: job.project,
         file: artifacts_file,
         file_type: params['artifact_type'],
@@ -21,34 +35,51 @@ module Ci
         file_sha256: artifacts_file.sha256,
         expire_in: expire_in)
 
-      if metadata_file
-        job.job_artifacts.build(
-          project: job.project,
-          file: metadata_file,
-          file_type: :metadata,
-          file_format: :gzip,
-          file_sha256: metadata_file.sha256,
-          expire_in: expire_in)
+      artifact_metadata = if metadata_file
+                            Ci::JobArtifact.new(
+                              job_id: job.id,
+                              project: job.project,
+                              file: metadata_file,
+                              file_type: :metadata,
+                              file_format: :gzip,
+                              file_sha256: metadata_file.sha256,
+                              expire_in: expire_in)
+                          end
+
+      [artifact, artifact_metadata]
+    end
+
+    def parse_artifact(job, artifact)
+      unless Feature.enabled?(:ci_synchronous_artifact_parsing, job.project, default_enabled: true)
+        return success
       end
 
-      if job.update(artifacts_expire_in: expire_in)
-        success
-      else
-        error(job.errors.messages, :bad_request)
+      case artifact.file_type
+      when 'dotenv' then parse_dotenv_artifact(job, artifact)
+      else success
       end
+    end
 
-    rescue ActiveRecord::RecordNotUnique => error
-      return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
+    def persist_artifact(job, artifact, artifact_metadata)
+      Ci::JobArtifact.transaction do
+        artifact.save!
+        artifact_metadata&.save!
+
+        # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future.
+        job.update_column(:artifacts_expire_at, artifact.expire_at)
+      end
 
+      success
+    rescue ActiveRecord::RecordNotUnique => error
       track_exception(error, job, params)
       error('another artifact of the same type already exists', :bad_request)
     rescue *OBJECT_STORAGE_ERRORS => error
       track_exception(error, job, params)
       error(error.message, :service_unavailable)
+    rescue => error
+      error(error.message, :bad_request)
     end
 
-    private
-
     def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file)
       existing_artifact = job.job_artifacts.find_by_file_type(artifact_type)
       return false unless existing_artifact
@@ -63,5 +94,9 @@ module Ci
         uploading_type: params['artifact_type']
       )
     end
+
+    def parse_dotenv_artifact(job, artifact)
+      Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact)
+    end
   end
 end
diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb
new file mode 100644
index 00000000000..fcbdc94c097
--- /dev/null
+++ b/app/services/ci/parse_dotenv_artifact_service.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Ci
+  class ParseDotenvArtifactService < ::BaseService
+    MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes
+    MAX_ACCEPTABLE_VARIABLES_COUNT = 10
+
+    SizeLimitError = Class.new(StandardError)
+    ParserError = Class.new(StandardError)
+
+    def execute(artifact)
+      validate!(artifact)
+
+      variables = parse!(artifact)
+      Ci::JobVariable.bulk_insert!(variables)
+
+      success
+    rescue SizeLimitError, ParserError, ActiveRecord::RecordInvalid => error
+      Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
+      error(error.message, :bad_request)
+    end
+
+    private
+
+    def validate!(artifact)
+      unless artifact&.dotenv?
+        raise ArgumentError, 'Artifact is not dotenv file type'
+      end
+
+      unless artifact.file.size < MAX_ACCEPTABLE_DOTENV_SIZE
+        raise SizeLimitError,
+          "Dotenv Artifact Too Big. Maximum Allowable Size: #{MAX_ACCEPTABLE_DOTENV_SIZE}"
+      end
+    end
+
+    def parse!(artifact)
+      variables = []
+
+      artifact.each_blob do |blob|
+        blob.each_line do |line|
+          key, value = scan_line!(line)
+
+          variables << Ci::JobVariable.new(job_id: artifact.job_id,
+            source: :dotenv, key: key, value: value)
+        end
+      end
+
+      if variables.size > MAX_ACCEPTABLE_VARIABLES_COUNT
+        raise SizeLimitError,
+          "Dotenv files cannot have more than #{MAX_ACCEPTABLE_VARIABLES_COUNT} variables"
+      end
+
+      variables
+    end
+
+    def scan_line!(line)
+      result = line.scan(/^(.*)=(.*)$/).last
+
+      raise ParserError, 'Invalid Format' if result.nil?
+
+      result.each(&:strip!)
+    end
+  end
+end