Add latest changes from gitlab-org/gitlab@master

12 files changed, 238 insertions, 99 deletions

diff --git a/app/assets/javascripts/blob/pdf/index.js b/app/assets/javascripts/blob/pdf/index.js
index 19778d07983..218987585b4 100644
--- a/app/assets/javascripts/blob/pdf/index.js
+++ b/app/assets/javascripts/blob/pdf/index.js
@@ -1,57 +1,17 @@
 import Vue from 'vue';
-import pdfLab from '../../pdf/index.vue';
-import { GlLoadingIcon } from '@gitlab/ui';
+import PdfViewer from './pdf_viewer.vue';
 
 export default () => {
   const el = document.getElementById('js-pdf-viewer');
 
   return new Vue({
     el,
-    components: {
-      pdfLab,
-      GlLoadingIcon,
+    render(createElement) {
+      return createElement(PdfViewer, {
+        props: {
+          pdf: el.dataset.endpoint,
+        },
+      });
     },
-    data() {
-      return {
-        error: false,
-        loadError: false,
-        loading: true,
-        pdf: el.dataset.endpoint,
-      };
-    },
-    methods: {
-      onLoad() {
-        this.loading = false;
-      },
-      onError(error) {
-        this.loading = false;
-        this.loadError = true;
-        this.error = error;
-      },
-    },
-    template: `
-      <div class="js-pdf-viewer container-fluid md prepend-top-default append-bottom-default">
-        <div
-          class="text-center loading"
-          v-if="loading && !error">
-          <gl-loading-icon class="mt-5" size="lg"/>
-        </div>
-        <pdf-lab
-          v-if="!loadError"
-          :pdf="pdf"
-          @pdflabload="onLoad"
-          @pdflaberror="onError" />
-        <p
-          class="text-center"
-          v-if="error">
-          <span v-if="loadError">
-            An error occurred while loading the file. Please try again later.
-          </span>
-          <span v-else>
-            An error occurred while decoding the file.
-          </span>
-        </p>
-      </div>
-    `,
   });
 };
diff --git a/app/assets/javascripts/blob/pdf/pdf_viewer.vue b/app/assets/javascripts/blob/pdf/pdf_viewer.vue
new file mode 100644
index 00000000000..5eaddfc099a
--- /dev/null
+++ b/app/assets/javascripts/blob/pdf/pdf_viewer.vue
@@ -0,0 +1,49 @@
+<script>
+import PdfLab from '../../pdf/index.vue';
+import { GlLoadingIcon } from '@gitlab/ui';
+
+export default {
+  components: {
+    PdfLab,
+    GlLoadingIcon,
+  },
+  props: {
+    pdf: {
+      type: String,
+      required: true,
+    },
+  },
+  data() {
+    return {
+      error: false,
+      loadError: false,
+      loading: true,
+    };
+  },
+  methods: {
+    onLoad() {
+      this.loading = false;
+    },
+    onError(error) {
+      this.loading = false;
+      this.loadError = true;
+      this.error = error;
+    },
+  },
+};
+</script>
+
+<template>
+  <div class="js-pdf-viewer container-fluid md prepend-top-default append-bottom-default">
+    <div v-if="loading && !error" class="text-center loading">
+      <gl-loading-icon class="mt-5" size="lg" />
+    </div>
+    <pdf-lab v-if="!loadError" :pdf="pdf" @pdflabload="onLoad" @pdflaberror="onError" />
+    <p v-if="error" class="text-center">
+      <span v-if="loadError" ref="loadError">
+        {{ __('An error occurred while loading the file. Please try again later.') }}
+      </span>
+      <span v-else>{{ __('An error occurred while decoding the file.') }}</span>
+    </p>
+  </div>
+</template>
diff --git a/app/assets/javascripts/lib/utils/common_utils.js b/app/assets/javascripts/lib/utils/common_utils.js
index dd5a52fe1ce..abecfba5718 100644
--- a/app/assets/javascripts/lib/utils/common_utils.js
+++ b/app/assets/javascripts/lib/utils/common_utils.js
@@ -8,6 +8,7 @@ import axios from './axios_utils';
 import { getLocationHash } from './url_utility';
 import { convertToCamelCase, convertToSnakeCase } from './text_utility';
 import { isObject } from './type_utility';
+import { isFunction } from 'lodash';
 
 export const getPagePath = (index = 0) => {
   const page = $('body').attr('data-page') || '';
@@ -667,30 +668,34 @@ export const spriteIcon = (icon, className = '') => {
 };
 
 /**
- * This method takes in object with snake_case property names
- * and returns a new object with camelCase property names
- *
- * Reasoning for this method is to ensure consistent property
- * naming conventions across JS code.
+ * @callback ConversionFunction
+ * @param {string} prop
+ */
+
+/**
+ * This function takes a conversion function as the first parameter
+ * and applies this function to each prop in the provided object.
  *
  * This method also supports additional params in `options` object
  *
+ * @param {ConversionFunction} conversionFunction - Function to apply to each prop of the object.
  * @param {Object} obj - Object to be converted.
  * @param {Object} options - Object containing additional options.
  * @param {boolean} options.deep - FLag to allow deep object converting
- * @param {Array[]} dropKeys - List of properties to discard while building new object
- * @param {Array[]} ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object
+ * @param {Array[]} options.dropKeys - List of properties to discard while building new object
+ * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object
  */
-export const convertObjectPropsToCamelCase = (obj = {}, options = {}) => {
-  if (obj === null) {
+export const convertObjectProps = (conversionFunction, obj = {}, options = {}) => {
+  if (!isFunction(conversionFunction) || obj === null) {
     return {};
   }
 
-  const initial = Array.isArray(obj) ? [] : {};
   const { deep = false, dropKeys = [], ignoreKeyNames = [] } = options;
 
+  const isObjParameterArray = Array.isArray(obj);
+  const initialValue = isObjParameterArray ? [] : {};
+
   return Object.keys(obj).reduce((acc, prop) => {
-    const result = acc;
     const val = obj[prop];
 
     // Drop properties from new object if
@@ -702,34 +707,54 @@ export const convertObjectPropsToCamelCase = (obj = {}, options = {}) => {
     // Skip converting properties in new object
     // if there are any mentioned in options
     if (ignoreKeyNames.indexOf(prop) > -1) {
-      result[prop] = obj[prop];
+      acc[prop] = val;
       return acc;
     }
 
     if (deep && (isObject(val) || Array.isArray(val))) {
-      result[convertToCamelCase(prop)] = convertObjectPropsToCamelCase(val, options);
+      if (isObjParameterArray) {
+        acc[prop] = convertObjectProps(conversionFunction, val, options);
+      } else {
+        acc[conversionFunction(prop)] = convertObjectProps(conversionFunction, val, options);
+      }
     } else {
-      result[convertToCamelCase(prop)] = obj[prop];
+      acc[conversionFunction(prop)] = val;
     }
     return acc;
-  }, initial);
+  }, initialValue);
 };
 
 /**
+ * This method takes in object with snake_case property names
+ * and returns a new object with camelCase property names
+ *
+ * Reasoning for this method is to ensure consistent property
+ * naming conventions across JS code.
+ *
+ * This method also supports additional params in `options` object
+ *
+ * @param {Object} obj - Object to be converted.
+ * @param {Object} options - Object containing additional options.
+ * @param {boolean} options.deep - FLag to allow deep object converting
+ * @param {Array[]} options.dropKeys - List of properties to discard while building new object
+ * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object
+ */
+export const convertObjectPropsToCamelCase = (obj = {}, options = {}) =>
+  convertObjectProps(convertToCamelCase, obj, options);
+
+/**
  * Converts all the object keys to snake case
  *
- * @param {Object} obj    Object to transform
- * @returns {Object}
+ * This method also supports additional params in `options` object
+ *
+ * @param {Object} obj - Object to be converted.
+ * @param {Object} options - Object containing additional options.
+ * @param {boolean} options.deep - FLag to allow deep object converting
+ * @param {Array[]} options.dropKeys - List of properties to discard while building new object
+ * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object
  */
-// Follow up to add additional options param:
-// https://gitlab.com/gitlab-org/gitlab/issues/39173
-export const convertObjectPropsToSnakeCase = (obj = {}) =>
-  obj
-    ? Object.entries(obj).reduce(
-        (acc, [key, value]) => ({ ...acc, [convertToSnakeCase(key)]: value }),
-        {},
-      )
-    : {};
+export const convertObjectPropsToSnakeCase = (obj = {}, options = {}) =>
+  convertObjectProps(convertToSnakeCase, obj, options);
 
 export const imagePath = imgUrl =>
   `${gon.asset_host || ''}${gon.relative_url_root || ''}/assets/${imgUrl}`;
diff --git a/app/controllers/groups/settings/ci_cd_controller.rb b/app/controllers/groups/settings/ci_cd_controller.rb
index ffa3f2c3364..3d347429398 100644
--- a/app/controllers/groups/settings/ci_cd_controller.rb
+++ b/app/controllers/groups/settings/ci_cd_controller.rb
@@ -7,7 +7,7 @@ module Groups
       before_action :authorize_admin_group!
       before_action :authorize_update_max_artifacts_size!, only: [:update]
       before_action do
-        push_frontend_feature_flag(:new_variables_ui, @group)
+        push_frontend_feature_flag(:new_variables_ui, @group, default_enabled: true)
       end
       before_action :define_variables, only: [:show, :create_deploy_token]
 
diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb
index bea24d2b204..af185887a8c 100644
--- a/app/controllers/projects/merge_requests_controller.rb
+++ b/app/controllers/projects/merge_requests_controller.rb
@@ -21,7 +21,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
   before_action only: [:show] do
     push_frontend_feature_flag(:diffs_batch_load, @project, default_enabled: true)
     push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true)
-    push_frontend_feature_flag(:single_mr_diff_view, @project)
+    push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true)
     push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
   end
 
diff --git a/app/controllers/projects/settings/ci_cd_controller.rb b/app/controllers/projects/settings/ci_cd_controller.rb
index aac6ecb07e4..43c798bfc6e 100644
--- a/app/controllers/projects/settings/ci_cd_controller.rb
+++ b/app/controllers/projects/settings/ci_cd_controller.rb
@@ -6,7 +6,7 @@ module Projects
       before_action :authorize_admin_pipeline!
       before_action :define_variables
       before_action do
-        push_frontend_feature_flag(:new_variables_ui, @project)
+        push_frontend_feature_flag(:new_variables_ui, @project, default_enabled: true)
       end
 
       def show
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index f9a5f713814..38730357593 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -28,7 +28,8 @@ module Ci
       license_scanning: 'gl-license-scanning-report.json',
       performance: 'performance.json',
       metrics: 'metrics.txt',
-      lsif: 'lsif.json'
+      lsif: 'lsif.json',
+      dotenv: '.env'
     }.freeze
 
     INTERNAL_TYPES = {
@@ -43,6 +44,7 @@ module Ci
       metrics_referee: :gzip,
       network_referee: :gzip,
       lsif: :gzip,
+      dotenv: :gzip,
 
       # All these file formats use `raw` as we need to store them uncompressed
       # for Frontend to fetch the files and do analysis
@@ -118,7 +120,8 @@ module Ci
       metrics: 12, ## EE-specific
       metrics_referee: 13, ## runner referees
       network_referee: 14, ## runner referees
-      lsif: 15 # LSIF data for code navigation
+      lsif: 15, # LSIF data for code navigation
+      dotenv: 16
     }
 
     enum file_format: {
diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb
index 862a0bc1299..f2968c037c7 100644
--- a/app/models/ci/job_variable.rb
+++ b/app/models/ci/job_variable.rb
@@ -4,11 +4,14 @@ module Ci
   class JobVariable < ApplicationRecord
     extend Gitlab::Ci::Model
     include NewHasVariable
+    include BulkInsertSafe
 
     belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
 
     alias_attribute :secret_value, :value
 
-    validates :key, uniqueness: { scope: :job_id }
+    validates :key, uniqueness: { scope: :job_id }, unless: :dotenv_source?
+
+    enum source: { internal: 0, dotenv: 1 }, _suffix: true
   end
 end
diff --git a/app/serializers/diff_file_entity.rb b/app/serializers/diff_file_entity.rb
index c3826692c52..45c16aabe9e 100644
--- a/app/serializers/diff_file_entity.rb
+++ b/app/serializers/diff_file_entity.rb
@@ -67,14 +67,14 @@ class DiffFileEntity < DiffFileBaseEntity
   private
 
   def parallel_diff_view?(options, diff_file)
-    return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project)
+    return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project, default_enabled: true)
 
     # If we're not rendering inline, we must be rendering parallel
     !inline_diff_view?(options, diff_file)
   end
 
   def inline_diff_view?(options, diff_file)
-    return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project)
+    return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project, default_enabled: true)
 
     # If nothing is present, inline will be the default.
     options.fetch(:diff_view, :inline).to_sym == :inline
diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb
index 3aa2b16bc73..d207c215618 100644
--- a/app/services/ci/create_job_artifacts_service.rb
+++ b/app/services/ci/create_job_artifacts_service.rb
@@ -10,10 +10,24 @@ module Ci
     ].freeze
 
     def execute(job, artifacts_file, params, metadata_file: nil)
+      return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
+
+      artifact, artifact_metadata = build_artifact(job, artifacts_file, params, metadata_file)
+      result = parse_artifact(job, artifact)
+
+      return result unless result[:status] == :success
+
+      persist_artifact(job, artifact, artifact_metadata)
+    end
+
+    private
+
+    def build_artifact(job, artifacts_file, params, metadata_file)
       expire_in = params['expire_in'] ||
         Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
 
-      job.job_artifacts.build(
+      artifact = Ci::JobArtifact.new(
+        job_id: job.id,
         project: job.project,
         file: artifacts_file,
         file_type: params['artifact_type'],
@@ -21,34 +35,51 @@ module Ci
         file_sha256: artifacts_file.sha256,
         expire_in: expire_in)
 
-      if metadata_file
-        job.job_artifacts.build(
-          project: job.project,
-          file: metadata_file,
-          file_type: :metadata,
-          file_format: :gzip,
-          file_sha256: metadata_file.sha256,
-          expire_in: expire_in)
+      artifact_metadata = if metadata_file
+                            Ci::JobArtifact.new(
+                              job_id: job.id,
+                              project: job.project,
+                              file: metadata_file,
+                              file_type: :metadata,
+                              file_format: :gzip,
+                              file_sha256: metadata_file.sha256,
+                              expire_in: expire_in)
+                          end
+
+      [artifact, artifact_metadata]
+    end
+
+    def parse_artifact(job, artifact)
+      unless Feature.enabled?(:ci_synchronous_artifact_parsing, job.project, default_enabled: true)
+        return success
       end
 
-      if job.update(artifacts_expire_in: expire_in)
-        success
-      else
-        error(job.errors.messages, :bad_request)
+      case artifact.file_type
+      when 'dotenv' then parse_dotenv_artifact(job, artifact)
+      else success
       end
+    end
 
-    rescue ActiveRecord::RecordNotUnique => error
-      return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file)
+    def persist_artifact(job, artifact, artifact_metadata)
+      Ci::JobArtifact.transaction do
+        artifact.save!
+        artifact_metadata&.save!
+
+        # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future.
+        job.update_column(:artifacts_expire_at, artifact.expire_at)
+      end
 
+      success
+    rescue ActiveRecord::RecordNotUnique => error
       track_exception(error, job, params)
       error('another artifact of the same type already exists', :bad_request)
     rescue *OBJECT_STORAGE_ERRORS => error
       track_exception(error, job, params)
       error(error.message, :service_unavailable)
+    rescue => error
+      error(error.message, :bad_request)
     end
 
-    private
-
     def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file)
       existing_artifact = job.job_artifacts.find_by_file_type(artifact_type)
       return false unless existing_artifact
@@ -63,5 +94,9 @@ module Ci
         uploading_type: params['artifact_type']
       )
     end
+
+    def parse_dotenv_artifact(job, artifact)
+      Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact)
+    end
   end
 end
diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb
new file mode 100644
index 00000000000..fcbdc94c097
--- /dev/null
+++ b/app/services/ci/parse_dotenv_artifact_service.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Ci
+  class ParseDotenvArtifactService < ::BaseService
+    MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes
+    MAX_ACCEPTABLE_VARIABLES_COUNT = 10
+
+    SizeLimitError = Class.new(StandardError)
+    ParserError = Class.new(StandardError)
+
+    def execute(artifact)
+      validate!(artifact)
+
+      variables = parse!(artifact)
+      Ci::JobVariable.bulk_insert!(variables)
+
+      success
+    rescue SizeLimitError, ParserError, ActiveRecord::RecordInvalid => error
+      Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
+      error(error.message, :bad_request)
+    end
+
+    private
+
+    def validate!(artifact)
+      unless artifact&.dotenv?
+        raise ArgumentError, 'Artifact is not dotenv file type'
+      end
+
+      unless artifact.file.size < MAX_ACCEPTABLE_DOTENV_SIZE
+        raise SizeLimitError,
+          "Dotenv Artifact Too Big. Maximum Allowable Size: #{MAX_ACCEPTABLE_DOTENV_SIZE}"
+      end
+    end
+
+    def parse!(artifact)
+      variables = []
+
+      artifact.each_blob do |blob|
+        blob.each_line do |line|
+          key, value = scan_line!(line)
+
+          variables << Ci::JobVariable.new(job_id: artifact.job_id,
+            source: :dotenv, key: key, value: value)
+        end
+      end
+
+      if variables.size > MAX_ACCEPTABLE_VARIABLES_COUNT
+        raise SizeLimitError,
+          "Dotenv files cannot have more than #{MAX_ACCEPTABLE_VARIABLES_COUNT} variables"
+      end
+
+      variables
+    end
+
+    def scan_line!(line)
+      result = line.scan(/^(.*)=(.*)$/).last
+
+      raise ParserError, 'Invalid Format' if result.nil?
+
+      result.each(&:strip!)
+    end
+  end
+end
diff --git a/app/views/ci/variables/_index.html.haml b/app/views/ci/variables/_index.html.haml
index f11c730eba6..aadb2c62d83 100644
--- a/app/views/ci/variables/_index.html.haml
+++ b/app/views/ci/variables/_index.html.haml
@@ -5,7 +5,7 @@
     - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('ci/variables/README', anchor: 'protected-variables') }
     = s_('Environment variables are configured by your administrator to be %{link_start}protected%{link_end} by default').html_safe % { link_start: link_start, link_end: '</a>'.html_safe }
 
-- if Feature.enabled?(:new_variables_ui, @project || @group)
+- if Feature.enabled?(:new_variables_ui, @project || @group, default_enabled: true)
   - is_group = !@group.nil?
 
   #js-ci-project-variables{ data: { endpoint: save_endpoint, project_id: @project&.id || '', group: is_group.to_s, maskable_regex: ci_variable_maskable_regex} }