diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-14 00:09:38 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-14 00:09:38 +0300 |
commit | 232e0a31f1e5d5b3a788dfc3dba8f8d41df36bf9 (patch) | |
tree | a2b11b9a805ef1165d8730934ba4a4f801f31870 /app/services/ci | |
parent | 00fa950a34b1c94617110b150b8b2517d5241249 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/services/ci')
-rw-r--r-- | app/services/ci/create_job_artifacts_service.rb | 69 | ||||
-rw-r--r-- | app/services/ci/parse_dotenv_artifact_service.rb | 64 |
2 files changed, 116 insertions, 17 deletions
diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb index 3aa2b16bc73..d207c215618 100644 --- a/app/services/ci/create_job_artifacts_service.rb +++ b/app/services/ci/create_job_artifacts_service.rb @@ -10,10 +10,24 @@ module Ci ].freeze def execute(job, artifacts_file, params, metadata_file: nil) + return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file) + + artifact, artifact_metadata = build_artifact(job, artifacts_file, params, metadata_file) + result = parse_artifact(job, artifact) + + return result unless result[:status] == :success + + persist_artifact(job, artifact, artifact_metadata) + end + + private + + def build_artifact(job, artifacts_file, params, metadata_file) expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.job_artifacts.build( + artifact = Ci::JobArtifact.new( + job_id: job.id, project: job.project, file: artifacts_file, file_type: params['artifact_type'], @@ -21,34 +35,51 @@ module Ci file_sha256: artifacts_file.sha256, expire_in: expire_in) - if metadata_file - job.job_artifacts.build( - project: job.project, - file: metadata_file, - file_type: :metadata, - file_format: :gzip, - file_sha256: metadata_file.sha256, - expire_in: expire_in) + artifact_metadata = if metadata_file + Ci::JobArtifact.new( + job_id: job.id, + project: job.project, + file: metadata_file, + file_type: :metadata, + file_format: :gzip, + file_sha256: metadata_file.sha256, + expire_in: expire_in) + end + + [artifact, artifact_metadata] + end + + def parse_artifact(job, artifact) + unless Feature.enabled?(:ci_synchronous_artifact_parsing, job.project, default_enabled: true) + return success end - if job.update(artifacts_expire_in: expire_in) - success - else - error(job.errors.messages, :bad_request) + case artifact.file_type + when 'dotenv' then parse_dotenv_artifact(job, artifact) + else success end + end - rescue ActiveRecord::RecordNotUnique => error - return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file) + def persist_artifact(job, artifact, artifact_metadata) + Ci::JobArtifact.transaction do + artifact.save! + artifact_metadata&.save! + + # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future. + job.update_column(:artifacts_expire_at, artifact.expire_at) + end + success + rescue ActiveRecord::RecordNotUnique => error track_exception(error, job, params) error('another artifact of the same type already exists', :bad_request) rescue *OBJECT_STORAGE_ERRORS => error track_exception(error, job, params) error(error.message, :service_unavailable) + rescue => error + error(error.message, :bad_request) end - private - def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file) existing_artifact = job.job_artifacts.find_by_file_type(artifact_type) return false unless existing_artifact @@ -63,5 +94,9 @@ module Ci uploading_type: params['artifact_type'] ) end + + def parse_dotenv_artifact(job, artifact) + Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact) + end end end diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb new file mode 100644 index 00000000000..fcbdc94c097 --- /dev/null +++ b/app/services/ci/parse_dotenv_artifact_service.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Ci + class ParseDotenvArtifactService < ::BaseService + MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes + MAX_ACCEPTABLE_VARIABLES_COUNT = 10 + + SizeLimitError = Class.new(StandardError) + ParserError = Class.new(StandardError) + + def execute(artifact) + validate!(artifact) + + variables = parse!(artifact) + Ci::JobVariable.bulk_insert!(variables) + + success + rescue SizeLimitError, ParserError, ActiveRecord::RecordInvalid => error + Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id) + error(error.message, :bad_request) + end + + private + + def validate!(artifact) + unless artifact&.dotenv? + raise ArgumentError, 'Artifact is not dotenv file type' + end + + unless artifact.file.size < MAX_ACCEPTABLE_DOTENV_SIZE + raise SizeLimitError, + "Dotenv Artifact Too Big. Maximum Allowable Size: #{MAX_ACCEPTABLE_DOTENV_SIZE}" + end + end + + def parse!(artifact) + variables = [] + + artifact.each_blob do |blob| + blob.each_line do |line| + key, value = scan_line!(line) + + variables << Ci::JobVariable.new(job_id: artifact.job_id, + source: :dotenv, key: key, value: value) + end + end + + if variables.size > MAX_ACCEPTABLE_VARIABLES_COUNT + raise SizeLimitError, + "Dotenv files cannot have more than #{MAX_ACCEPTABLE_VARIABLES_COUNT} variables" + end + + variables + end + + def scan_line!(line) + result = line.scan(/^(.*)=(.*)$/).last + + raise ParserError, 'Invalid Format' if result.nil? + + result.each(&:strip!) + end + end +end |