Add latest changes from gitlab-org/gitlab@master

15 files changed, 215 insertions, 129 deletions

diff --git a/app/assets/javascripts/ide/components/repo_editor.vue b/app/assets/javascripts/ide/components/repo_editor.vue
index 2a8ed076ae2..55290ba7779 100644
--- a/app/assets/javascripts/ide/components/repo_editor.vue
+++ b/app/assets/javascripts/ide/components/repo_editor.vue
@@ -14,7 +14,6 @@ import Editor from '../lib/editor';
 import FileTemplatesBar from './file_templates/bar.vue';
 import { __ } from '~/locale';
 import { extractMarkdownImagesFromEntries } from '../stores/utils';
-import { addFinalNewline } from '../utils';
 
 export default {
   components: {
@@ -32,7 +31,6 @@ export default {
     return {
       content: '',
       images: {},
-      addFinalNewline: true,
     };
   },
   computed: {
@@ -253,10 +251,8 @@ export default {
 
         const monacoModel = model.getModel();
         const content = monacoModel.getValue();
-        this.changeFileContent({
-          path: file.path,
-          content: this.addFinalNewline ? addFinalNewline(content, monacoModel.getEOL()) : content,
-        });
+        this.changeFileContent({ path: file.path, content });
+        this.setFileEOL({ eol: this.model.eol });
       });
 
       // Handle Cursor Position
diff --git a/app/assets/javascripts/ide/lib/common/model.js b/app/assets/javascripts/ide/lib/common/model.js
index a15f04075d9..4c743076fb2 100644
--- a/app/assets/javascripts/ide/lib/common/model.js
+++ b/app/assets/javascripts/ide/lib/common/model.js
@@ -1,6 +1,8 @@
 import { editor as monacoEditor, Uri } from 'monaco-editor';
 import Disposable from './disposable';
 import eventHub from '../../eventhub';
+import { trimTrailingWhitespace, insertFinalNewline } from '../../utils';
+import { defaultModelOptions } from '../editor_options';
 
 export default class Model {
   constructor(file, head = null) {
@@ -8,6 +10,7 @@ export default class Model {
     this.file = file;
     this.head = head;
     this.content = file.content !== '' || file.deleted ? file.content : file.raw;
+    this.options = { ...defaultModelOptions };
 
     this.disposable.add(
       (this.originalModel = monacoEditor.createModel(
@@ -94,8 +97,32 @@ export default class Model {
     this.getModel().setValue(content);
   }
 
+  updateOptions(obj = {}) {
+    Object.assign(this.options, obj);
+    this.model.updateOptions(obj);
+    this.applyCustomOptions();
+  }
+
+  applyCustomOptions() {
+    this.updateNewContent(
+      Object.entries(this.options).reduce((content, [key, value]) => {
+        switch (key) {
+          case 'endOfLine':
+            this.model.pushEOL(value);
+            return this.model.getValue();
+          case 'insertFinalNewline':
+            return value ? insertFinalNewline(content) : content;
+          case 'trimTrailingWhitespace':
+            return value ? trimTrailingWhitespace(content) : content;
+          default:
+            return content;
+        }
+      }, this.model.getValue()),
+    );
+  }
+
   dispose() {
-    this.disposable.dispose();
+    if (!this.model.isDisposed()) this.applyCustomOptions();
 
     this.events.forEach(cb => {
       if (typeof cb === 'function') cb();
@@ -106,5 +133,7 @@ export default class Model {
     eventHub.$off(`editor.update.model.dispose.${this.file.key}`, this.dispose);
     eventHub.$off(`editor.update.model.content.${this.file.key}`, this.updateContent);
     eventHub.$off(`editor.update.model.new.content.${this.file.key}`, this.updateNewContent);
+
+    this.disposable.dispose();
   }
 }
diff --git a/app/assets/javascripts/ide/lib/diff/controller.js b/app/assets/javascripts/ide/lib/diff/controller.js
index 234a7f903a1..35fcda6a6c5 100644
--- a/app/assets/javascripts/ide/lib/diff/controller.js
+++ b/app/assets/javascripts/ide/lib/diff/controller.js
@@ -50,10 +50,15 @@ export default class DirtyDiffController {
   }
 
   computeDiff(model) {
+    const originalModel = model.getOriginalModel();
+    const newModel = model.getModel();
+
+    if (originalModel.isDisposed() || newModel.isDisposed()) return;
+
     this.dirtyDiffWorker.postMessage({
       path: model.path,
-      originalContent: model.getOriginalModel().getValue(),
-      newContent: model.getModel().getValue(),
+      originalContent: originalModel.getValue(),
+      newContent: newModel.getValue(),
     });
   }
 
diff --git a/app/assets/javascripts/ide/lib/diff/diff.js b/app/assets/javascripts/ide/lib/diff/diff.js
index 29e29d7fcd3..3a456b7c4d6 100644
--- a/app/assets/javascripts/ide/lib/diff/diff.js
+++ b/app/assets/javascripts/ide/lib/diff/diff.js
@@ -1,8 +1,15 @@
 import { diffLines } from 'diff';
+import { defaultDiffOptions } from '../editor_options';
 
+// See: https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/20
 // eslint-disable-next-line import/prefer-default-export
 export const computeDiff = (originalContent, newContent) => {
-  const changes = diffLines(originalContent, newContent);
+  // prevent EOL changes from highlighting the entire file
+  const changes = diffLines(
+    originalContent.replace(/\r\n/g, '\n'),
+    newContent.replace(/\r\n/g, '\n'),
+    defaultDiffOptions,
+  );
 
   let lineNumber = 1;
   return changes.reduce((acc, change) => {
diff --git a/app/assets/javascripts/ide/lib/editor.js b/app/assets/javascripts/ide/lib/editor.js
index 25224abd77c..141efd86884 100644
--- a/app/assets/javascripts/ide/lib/editor.js
+++ b/app/assets/javascripts/ide/lib/editor.js
@@ -5,7 +5,7 @@ import DecorationsController from './decorations/controller';
 import DirtyDiffController from './diff/controller';
 import Disposable from './common/disposable';
 import ModelManager from './common/model_manager';
-import editorOptions, { defaultEditorOptions } from './editor_options';
+import { editorOptions, defaultEditorOptions, defaultDiffEditorOptions } from './editor_options';
 import { themes } from './themes';
 import languages from './languages';
 import keymap from './keymap.json';
@@ -73,8 +73,7 @@ export default class Editor {
       this.disposable.add(
         (this.instance = monacoEditor.createDiffEditor(domElement, {
           ...this.options,
-          quickSuggestions: false,
-          occurrencesHighlight: false,
+          ...defaultDiffEditorOptions,
           renderSideBySide: Editor.renderSideBySide(domElement),
           readOnly,
           renderLineHighlight: readOnly ? 'all' : 'none',
diff --git a/app/assets/javascripts/ide/lib/editor_options.js b/app/assets/javascripts/ide/lib/editor_options.js
index dac2a8e8b51..422b2b9776c 100644
--- a/app/assets/javascripts/ide/lib/editor_options.js
+++ b/app/assets/javascripts/ide/lib/editor_options.js
@@ -9,7 +9,23 @@ export const defaultEditorOptions = {
   wordWrap: 'on',
 };
 
-export default [
+export const defaultDiffOptions = {
+  ignoreWhitespace: false,
+};
+
+export const defaultDiffEditorOptions = {
+  quickSuggestions: false,
+  occurrencesHighlight: false,
+  ignoreTrimWhitespace: false,
+};
+
+export const defaultModelOptions = {
+  endOfLine: 0,
+  insertFinalNewline: true,
+  trimTrailingWhitespace: false,
+};
+
+export const editorOptions = [
   {
     readOnly: model => Boolean(model.file.file_lock),
     quickSuggestions: model => !(model.language === 'markdown'),
diff --git a/app/assets/javascripts/ide/utils.js b/app/assets/javascripts/ide/utils.js
index 64be1f59887..2d1a7c24c59 100644
--- a/app/assets/javascripts/ide/utils.js
+++ b/app/assets/javascripts/ide/utils.js
@@ -77,7 +77,11 @@ export function registerLanguages(def, ...defs) {
 
 export const otherSide = side => (side === SIDE_RIGHT ? SIDE_LEFT : SIDE_RIGHT);
 
-export function addFinalNewline(content, eol = '\n') {
+export function trimTrailingWhitespace(content) {
+  return content.replace(/[^\S\r\n]+$/gm, '');
+}
+
+export function insertFinalNewline(content, eol = '\n') {
   return content.slice(-eol.length) !== eol ? `${content}${eol}` : content;
 }
 
diff --git a/app/controllers/admin/runners_controller.rb b/app/controllers/admin/runners_controller.rb
index 4639d8adfe0..2449fa3128c 100644
--- a/app/controllers/admin/runners_controller.rb
+++ b/app/controllers/admin/runners_controller.rb
@@ -4,7 +4,7 @@ class Admin::RunnersController < Admin::ApplicationController
   before_action :runner, except: [:index, :tag_list]
 
   def index
-    finder = Admin::RunnersFinder.new(params: params)
+    finder = Ci::RunnersFinder.new(current_user: current_user, params: params)
     @runners = finder.execute
     @active_runners_count = Ci::Runner.online.count
     @sort = finder.sort_key
diff --git a/app/finders/admin/runners_finder.rb b/app/finders/admin/runners_finder.rb
deleted file mode 100644
index b2799565f57..00000000000
--- a/app/finders/admin/runners_finder.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-class Admin::RunnersFinder < UnionFinder
-  NUMBER_OF_RUNNERS_PER_PAGE = 30
-
-  def initialize(params:)
-    @params = params
-  end
-
-  def execute
-    search!
-    filter_by_status!
-    filter_by_runner_type!
-    filter_by_tag_list!
-    sort!
-    paginate!
-
-    @runners.with_tags
-  end
-
-  def sort_key
-    if @params[:sort] == 'contacted_asc'
-      'contacted_asc'
-    else
-      'created_date'
-    end
-  end
-
-  private
-
-  def search!
-    @runners =
-      if @params[:search].present?
-        Ci::Runner.search(@params[:search])
-      else
-        Ci::Runner.all
-      end
-  end
-
-  def filter_by_status!
-    filter_by!(:status_status, Ci::Runner::AVAILABLE_STATUSES)
-  end
-
-  def filter_by_runner_type!
-    filter_by!(:type_type, Ci::Runner::AVAILABLE_TYPES)
-  end
-
-  def filter_by_tag_list!
-    tag_list = @params[:tag_name].presence
-
-    if tag_list
-      @runners = @runners.tagged_with(tag_list)
-    end
-  end
-
-  def sort!
-    @runners = @runners.order_by(sort_key)
-  end
-
-  def paginate!
-    @runners = @runners.page(@params[:page]).per(NUMBER_OF_RUNNERS_PER_PAGE)
-  end
-
-  def filter_by!(scope_name, available_scopes)
-    scope = @params[scope_name]
-
-    if scope.present? && available_scopes.include?(scope)
-      @runners = @runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend
-    end
-  end
-end
diff --git a/app/finders/ci/runners_finder.rb b/app/finders/ci/runners_finder.rb
new file mode 100644
index 00000000000..54d9d9522e0
--- /dev/null
+++ b/app/finders/ci/runners_finder.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Ci
+  class RunnersFinder < UnionFinder
+    include Gitlab::Allowable
+
+    NUMBER_OF_RUNNERS_PER_PAGE = 30
+
+    def initialize(current_user:, group: nil, params:)
+      @params = params
+      @group = group
+      @current_user = current_user
+    end
+
+    def execute
+      search!
+      filter_by_status!
+      filter_by_runner_type!
+      filter_by_tag_list!
+      sort!
+      paginate!
+
+      @runners.with_tags
+
+    rescue Gitlab::Access::AccessDeniedError
+      Ci::Runner.none
+    end
+
+    def sort_key
+      if @params[:sort] == 'contacted_asc'
+        'contacted_asc'
+      else
+        'created_date'
+      end
+    end
+
+    private
+
+    def search!
+      @group ? group_runners : all_runners
+
+      @runners = @runners.search(@params[:search]) if @params[:search].present?
+    end
+
+    def all_runners
+      raise Gitlab::Access::AccessDeniedError unless @current_user&.admin?
+
+      @runners = Ci::Runner.all
+    end
+
+    def group_runners
+      raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group)
+
+      # Getting all runners from the group itself and all its descendants
+      descentant_projects = Project.for_group_and_its_subgroups(@group)
+
+      @runners = Ci::Runner.belonging_to_group_or_project(@group.self_and_descendants, descentant_projects)
+    end
+
+    def filter_by_status!
+      filter_by!(:status_status, Ci::Runner::AVAILABLE_STATUSES)
+    end
+
+    def filter_by_runner_type!
+      filter_by!(:type_type, Ci::Runner::AVAILABLE_TYPES)
+    end
+
+    def filter_by_tag_list!
+      tag_list = @params[:tag_name].presence
+
+      if tag_list
+        @runners = @runners.tagged_with(tag_list)
+      end
+    end
+
+    def sort!
+      @runners = @runners.order_by(sort_key)
+    end
+
+    def paginate!
+      @runners = @runners.page(@params[:page]).per(NUMBER_OF_RUNNERS_PER_PAGE)
+    end
+
+    def filter_by!(scope_name, available_scopes)
+      scope = @params[scope_name]
+
+      if scope.present? && available_scopes.include?(scope)
+        @runners = @runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend
+      end
+    end
+  end
+end
diff --git a/app/finders/uploader_finder.rb b/app/finders/uploader_finder.rb
new file mode 100644
index 00000000000..0d1de0d56fd
--- /dev/null
+++ b/app/finders/uploader_finder.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class UploaderFinder
+  # Instantiates a a new FileUploader
+  # FileUploader can be opened via .open agnostic of storage type
+  # Arguments correspond to Upload.secret, Upload.model_type and Upload.file_path
+  # Returns a FileUploader with uploaded file retrieved into the object state
+  def initialize(project, secret, file_path)
+    @project = project
+    @secret = secret
+    @file_path = file_path
+  end
+
+  def execute
+    prevent_path_traversal_attack!
+    retrieve_file_state!
+
+    uploader
+  rescue ::Gitlab::Utils::PathTraversalAttackError
+    nil # no-op if for incorrect files
+  end
+
+  def prevent_path_traversal_attack!
+    Gitlab::Utils.check_path_traversal!(@file_path)
+  end
+
+  def retrieve_file_state!
+    uploader.retrieve_from_store!(@file_path)
+  end
+
+  def uploader
+    @uploader ||= FileUploader.new(@project, secret: @secret)
+  end
+end
diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb
index c674f76d229..557f3a63280 100644
--- a/app/models/ci/instance_variable.rb
+++ b/app/models/ci/instance_variable.rb
@@ -3,6 +3,7 @@
 module Ci
   class InstanceVariable < ApplicationRecord
     extend Gitlab::Ci::Model
+    extend Gitlab::ProcessMemoryCache::Helper
     include Ci::NewHasVariable
     include Ci::Maskable
 
@@ -13,7 +14,8 @@ module Ci
     }
 
     scope :unprotected, -> { where(protected: false) }
-    after_commit { self.class.touch_redis_cache_timestamp }
+
+    after_commit { self.class.invalidate_memory_cache(:ci_instance_variable_data) }
 
     class << self
       def all_cached
@@ -24,10 +26,6 @@ module Ci
         cached_data[:unprotected]
       end
 
-      def touch_redis_cache_timestamp(time = Time.current.to_f)
-        shared_backend.write(:ci_instance_variable_changed_at, time)
-      end
-
       private
 
       def cached_data
@@ -37,40 +35,6 @@ module Ci
           { all: all_records, unprotected: all_records.reject(&:protected?) }
         end
       end
-
-      def fetch_memory_cache(key, &payload)
-        cache = process_backend.read(key)
-
-        if cache && !stale_cache?(cache)
-          cache[:data]
-        else
-          store_cache(key, &payload)
-        end
-      end
-
-      def stale_cache?(cache_info)
-        shared_timestamp = shared_backend.read(:ci_instance_variable_changed_at)
-        return true unless shared_timestamp
-
-        shared_timestamp.to_f > cache_info[:cached_at].to_f
-      end
-
-      def store_cache(key)
-        data = yield
-        time = Time.current.to_f
-
-        process_backend.write(key, data: data, cached_at: time)
-        touch_redis_cache_timestamp(time)
-        data
-      end
-
-      def shared_backend
-        Rails.cache
-      end
-
-      def process_backend
-        Gitlab::ProcessMemoryCache.cache_backend
-      end
     end
   end
 end
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 03ff952b435..30ea383bd73 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -81,6 +81,17 @@ module Ci
       joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups })
     }
 
+    scope :belonging_to_group_or_project, -> (group_id, project_id) {
+      groups = ::Group.where(id: group_id)
+
+      group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups })
+      project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
+
+      union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql
+
+      from("(#{union_sql}) #{table_name}")
+    }
+
     scope :belonging_to_parent_group_of_project, -> (project_id) {
       project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
       hierarchy_groups = Gitlab::ObjectHierarchy.new(project_groups).base_and_ancestors
diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml
index c738493507f..409e602f306 100644
--- a/app/views/layouts/nav/sidebar/_project.html.haml
+++ b/app/views/layouts/nav/sidebar/_project.html.haml
@@ -319,7 +319,7 @@
 
       - if project_nav_tab? :snippets
         = nav_link(controller: :snippets) do
-          = link_to project_snippets_path(@project), class: 'shortcuts-snippets' do
+          = link_to project_snippets_path(@project), class: 'shortcuts-snippets', data: { qa_selector: 'snippets_link' } do
             .nav-icon-container
               = sprite_icon('snippet')
             %span.nav-item-name
diff --git a/app/views/shared/empty_states/_snippets.html.haml b/app/views/shared/empty_states/_snippets.html.haml
index efd9bceedc5..db8da50d868 100644
--- a/app/views/shared/empty_states/_snippets.html.haml
+++ b/app/views/shared/empty_states/_snippets.html.haml
@@ -3,7 +3,7 @@
 .row.empty-state.mt-0
   .col-12
     .svg-content
-      = image_tag 'illustrations/snippets_empty.svg'
+      = image_tag 'illustrations/snippets_empty.svg', data: { qa_selector: 'svg_content' }
     .text-content.text-center.pt-0
       - if current_user
         %h4
@@ -12,7 +12,7 @@
           = s_('SnippetsEmptyState|Store, share, and embed small pieces of code and text.')
         .mt-2<
           - if button_path
-            = link_to s_('SnippetsEmptyState|New snippet'), button_path, class: 'btn btn-success', title: s_('SnippetsEmptyState|New snippet'), id: 'new_snippet_link'
+            = link_to s_('SnippetsEmptyState|New snippet'), button_path, class: 'btn btn-success', title: s_('SnippetsEmptyState|New snippet'), id: 'new_snippet_link', data: { qa_selector: 'create_first_snippet_link' }
           = link_to s_('SnippetsEmptyState|Documentation'), help_page_path('user/snippets.md'), class: 'btn btn-default', title: s_('SnippetsEmptyState|Documentation')
       - else
         %h4.text-center= s_('SnippetsEmptyState|There are no snippets to show.')