Add latest changes from gitlab-org/gitlab@master

13 files changed, 258 insertions, 134 deletions

diff --git a/app/assets/javascripts/content_editor/services/hast_to_prosemirror_converter.js b/app/assets/javascripts/content_editor/services/hast_to_prosemirror_converter.js
index a0be6c70993..5d675fb4851 100644
--- a/app/assets/javascripts/content_editor/services/hast_to_prosemirror_converter.js
+++ b/app/assets/javascripts/content_editor/services/hast_to_prosemirror_converter.js
@@ -138,6 +138,10 @@ class HastToProseMirrorConverterState {
     return this.stack[this.stack.length - 1];
   }
 
+  get topNode() {
+    return this.findInStack((item) => item.type === 'node');
+  }
+
   /**
    * Detects if the node stack is empty
    */
@@ -179,7 +183,7 @@ class HastToProseMirrorConverterState {
    */
   addText(schema, text) {
     if (!text) return;
-    const nodes = this.top.content;
+    const nodes = this.topNode?.content;
     const last = nodes[nodes.length - 1];
     const node = schema.text(text, this.marks);
     const merged = maybeMerge(last, node);
@@ -189,57 +193,92 @@ class HastToProseMirrorConverterState {
     } else {
       nodes.push(node);
     }
-
-    this.closeMarks();
   }
 
   /**
    * Adds a mark to the set of marks stored temporarily
-   * until addText is called.
-   * @param {*} markType
-   * @param {*} attrs
+   * until an inline node is created.
+   * @param {https://prosemirror.net/docs/ref/#model.MarkType} schemaType Mark schema type
+   * @param {https://github.com/syntax-tree/hast#nodes} hastNode AST node that the mark is based on
+   * @param {Object} attrs Mark attributes
+   * @param {Object} factorySpec Specifications on how th mark should be created
    */
-  openMark(markType, attrs) {
-    this.marks = markType.create(attrs).addToSet(this.marks);
+  openMark(schemaType, hastNode, attrs, factorySpec) {
+    const mark = schemaType.create(attrs);
+    this.stack.push({
+      type: 'mark',
+      mark,
+      attrs,
+      hastNode,
+      factorySpec,
+    });
+
+    this.marks = mark.addToSet(this.marks);
   }
 
   /**
-   * Empties the temporary Mark set.
+   * Removes a mark from the list of active marks that
+   * are applied to inline nodes.
    */
-  closeMarks() {
-    this.marks = Mark.none;
+  closeMark() {
+    const { mark } = this.stack.pop();
+
+    this.marks = mark.removeFromSet(this.marks);
   }
 
   /**
    * Adds a node to the stack data structure.
    *
-   * @param {Schema.NodeType} type ProseMirror Schema for the node
-   * @param {HastNode} hastNode Hast node from which the ProseMirror node will be created
+   * @param {https://prosemirror.net/docs/ref/#model.NodeType} schemaType ProseMirror Schema for the node
+   * @param {https://github.com/syntax-tree/hast#nodes} hastNode Hast node from which the ProseMirror node will be created
    * @param {*} attrs Node’s attributes
    * @param {*} factorySpec The factory spec used to create the node factory
    */
-  openNode(type, hastNode, attrs, factorySpec) {
-    this.stack.push({ type, attrs, content: [], hastNode, factorySpec });
+  openNode(schemaType, hastNode, attrs, factorySpec) {
+    this.stack.push({
+      type: 'node',
+      schemaType,
+      attrs,
+      content: [],
+      hastNode,
+      factorySpec,
+    });
   }
 
   /**
    * Removes the top ProseMirror node from the
    * conversion stack and adds the node to the
    * previous element.
-   * @returns
    */
   closeNode() {
-    const { type, attrs, content } = this.stack.pop();
-    const node = type.createAndFill(attrs, content);
-
-    if (!node) return null;
-
-    if (this.marks.length) {
-      this.marks = Mark.none;
+    const { schemaType, attrs, content, factorySpec } = this.stack.pop();
+    const node =
+      factorySpec.type === 'inline' && this.marks.length
+        ? schemaType.createAndFill(attrs, content, this.marks)
+        : schemaType.createAndFill(attrs, content);
+
+    if (!node) {
+      /*
+      When the node returned by `createAndFill` is null is because the `content` passed as a parameter
+      doesn’t conform with the document schema. We are handling the most likely scenario here that happens
+      when a paragraph is inside another paragraph.
+
+      This scenario happens when the converter encounters a mark wrapping one or more paragraphs.
+      In this case, the converter will wrap the mark in a paragraph as well because ProseMirror does
+      not allow marks wrapping block nodes or being direct children of certain nodes like the root nodes
+      or list items.
+      */
+      if (
+        schemaType.name === 'paragraph' &&
+        content.some((child) => child.type.name === 'paragraph')
+      ) {
+        this.topNode.content.push(...content);
+      }
+      return null;
     }
 
     if (!this.empty) {
-      this.top.content.push(node);
+      this.topNode.content.push(node);
     }
 
     return node;
@@ -247,9 +286,27 @@ class HastToProseMirrorConverterState {
 
   closeUntil(hastNode) {
     while (hastNode !== this.top?.hastNode) {
-      this.closeNode();
+      if (this.top.type === 'node') {
+        this.closeNode();
+      } else {
+        this.closeMark();
+      }
     }
   }
+
+  buildDoc() {
+    let doc;
+
+    do {
+      if (this.top.type === 'node') {
+        doc = this.closeNode();
+      } else {
+        this.closeMark();
+      }
+    } while (!this.empty);
+
+    return doc;
+  }
 }
 
 /**
@@ -276,7 +333,7 @@ const createProseMirrorNodeFactories = (schema, proseMirrorFactorySpecs, source)
     },
     text: {
       selector: 'text',
-      handle: (state, hastNode) => {
+      handle: (state, hastNode, parent) => {
         const found = state.findInStack((node) => isFunction(node.factorySpec.processText));
         const { value: text } = hastNode;
 
@@ -284,6 +341,7 @@ const createProseMirrorNodeFactories = (schema, proseMirrorFactorySpecs, source)
           return;
         }
 
+        state.closeUntil(parent);
         state.addText(schema, found ? found.factorySpec.processText(text) : text);
       },
     },
@@ -320,11 +378,7 @@ const createProseMirrorNodeFactories = (schema, proseMirrorFactorySpecs, source)
     } else if (factory.type === 'mark') {
       const markType = schema.marks[proseMirrorName];
       factory.handle = (state, hastNode, parent) => {
-        state.openMark(markType, getAttrs(factory, hastNode, parent, source));
-
-        if (factory.inlineContent) {
-          state.addText(schema, hastNode.value);
-        }
+        state.openMark(markType, hastNode, getAttrs(factory, hastNode, parent, source), factory);
       };
     } else if (factory.type === 'ignore') {
       factory.handle = noop;
@@ -357,7 +411,7 @@ const findParent = (ancestors, parent) => {
   return ancestors[ancestors.length - 1];
 };
 
-const calcTextNodePosition = (textNode) => {
+const resolveNodePosition = (textNode) => {
   const { position, value, type } = textNode;
 
   if (type !== 'text' || (!position.start && !position.end) || (position.start && position.end)) {
@@ -418,7 +472,7 @@ const wrapInlineElements = (nodes, wrappableTags) =>
     const wrapper = {
       type: 'element',
       tagName: 'p',
-      position: calcTextNodePosition(child),
+      position: resolveNodePosition(child),
       children: [child],
       properties: { wrapper: true },
     };
@@ -580,11 +634,5 @@ export const createProseMirrorDocFromMdastTree = ({
     return factory.skipChildren === true ? SKIP : true;
   });
 
-  let doc;
-
-  do {
-    doc = state.closeNode();
-  } while (!state.empty);
-
-  return doc;
+  return state.buildDoc();
 };
diff --git a/app/assets/javascripts/main.js b/app/assets/javascripts/main.js
index a725ef972d4..21d5decb15b 100644
--- a/app/assets/javascripts/main.js
+++ b/app/assets/javascripts/main.js
@@ -115,34 +115,6 @@ function deferredInitialisation() {
     );
   }
 
-  const searchInputBox = document.querySelector('#search');
-  if (searchInputBox) {
-    searchInputBox.addEventListener(
-      'focus',
-      () => {
-        if (gon.features?.newHeaderSearch) {
-          import(/* webpackChunkName: 'globalSearch' */ '~/header_search')
-            .then(async ({ initHeaderSearchApp }) => {
-              // In case the user started searching before we bootstrapped, let's pass the search along.
-              const initialSearchValue = searchInputBox.value;
-              await initHeaderSearchApp(initialSearchValue);
-              // this is new #search input element. We need to re-find it.
-              document.querySelector('#search').focus();
-            })
-            .catch(() => {});
-        } else {
-          import(/* webpackChunkName: 'globalSearch' */ './search_autocomplete')
-            .then(({ default: initSearchAutocomplete }) => {
-              const searchDropdown = initSearchAutocomplete();
-              searchDropdown.onSearchInputFocus();
-            })
-            .catch(() => {});
-        }
-      },
-      { once: true },
-    );
-  }
-
   addSelectOnFocusBehaviour('.js-select-on-focus');
 
   const glTooltipDelay = localStorage.getItem('gl-tooltip-delay');
@@ -169,6 +141,36 @@ function deferredInitialisation() {
   }
 }
 
+// loading this inside requestIdleCallback is causing issues
+// see https://gitlab.com/gitlab-org/gitlab/-/issues/365746
+const searchInputBox = document.querySelector('#search');
+if (searchInputBox) {
+  searchInputBox.addEventListener(
+    'focus',
+    () => {
+      if (gon.features?.newHeaderSearch) {
+        import(/* webpackChunkName: 'globalSearch' */ '~/header_search')
+          .then(async ({ initHeaderSearchApp }) => {
+            // In case the user started searching before we bootstrapped, let's pass the search along.
+            const initialSearchValue = searchInputBox.value;
+            await initHeaderSearchApp(initialSearchValue);
+            // this is new #search input element. We need to re-find it.
+            document.querySelector('#search').focus();
+          })
+          .catch(() => {});
+      } else {
+        import(/* webpackChunkName: 'globalSearch' */ './search_autocomplete')
+          .then(({ default: initSearchAutocomplete }) => {
+            const searchDropdown = initSearchAutocomplete();
+            searchDropdown.onSearchInputFocus();
+          })
+          .catch(() => {});
+      }
+    },
+    { once: true },
+  );
+}
+
 const $body = $('body');
 const $document = $(document);
 const bootstrapBreakpoint = bp.getBreakpointSize();
diff --git a/app/assets/stylesheets/highlight/themes/monokai.scss b/app/assets/stylesheets/highlight/themes/monokai.scss
index c19189ef31b..f83bad152c5 100644
--- a/app/assets/stylesheets/highlight/themes/monokai.scss
+++ b/app/assets/stylesheets/highlight/themes/monokai.scss
@@ -115,7 +115,7 @@ $monokai-gh: #75715e;
   @include hljs-override('section', $monokai-gh);
   @include hljs-override('bullet', $monokai-n);
   @include hljs-override('subst', $monokai-p);
-  @include hljs-override('symbol', $monokai-ni);
+  @include hljs-override('symbol', $monokai-ss);
 
   // Line numbers
   .file-line-num {
diff --git a/app/assets/stylesheets/highlight/themes/white.scss b/app/assets/stylesheets/highlight/themes/white.scss
index 8698e448c94..b6ca26e9b39 100644
--- a/app/assets/stylesheets/highlight/themes/white.scss
+++ b/app/assets/stylesheets/highlight/themes/white.scss
@@ -2,6 +2,7 @@
   @import '../white_base';
 
   @include conflict-colors('white');
+  @include hljs-override('symbol', $white-ss);
 }
 
 :root {
diff --git a/app/models/packages/cleanup/policy.rb b/app/models/packages/cleanup/policy.rb
index d7df90a4ce0..c0585a52e6e 100644
--- a/app/models/packages/cleanup/policy.rb
+++ b/app/models/packages/cleanup/policy.rb
@@ -27,6 +27,10 @@ module Packages
         # fixed cadence of 12 hours
         self.next_run_at = Time.zone.now + 12.hours
       end
+
+      def keep_n_duplicated_package_files_disabled?
+        keep_n_duplicated_package_files == 'all'
+      end
     end
   end
 end
diff --git a/app/services/alert_management/alerts/update_service.rb b/app/services/alert_management/alerts/update_service.rb
index 6bdceb0f27b..f273e15b159 100644
--- a/app/services/alert_management/alerts/update_service.rb
+++ b/app/services/alert_management/alerts/update_service.rb
@@ -12,7 +12,6 @@ module AlertManagement
         @alert = alert
         @param_errors = []
         @status = params.delete(:status)
-        @status_change_reason = params.delete(:status_change_reason)
 
         super(project: alert.project, current_user: current_user, params: params)
       end
@@ -37,7 +36,7 @@ module AlertManagement
 
       private
 
-      attr_reader :alert, :param_errors, :status, :status_change_reason
+      attr_reader :alert, :param_errors, :status
 
       def allowed?
         current_user&.can?(:update_alert_management_alert, alert)
@@ -130,37 +129,16 @@ module AlertManagement
       def handle_status_change
         add_status_change_system_note
         resolve_todos if alert.resolved?
-        sync_to_incident if should_sync_to_incident?
       end
 
       def add_status_change_system_note
-        SystemNoteService.change_alert_status(alert, current_user, status_change_reason)
+        SystemNoteService.change_alert_status(alert, current_user)
       end
 
       def resolve_todos
         todo_service.resolve_todos_for_target(alert, current_user)
       end
 
-      def sync_to_incident
-        ::Issues::UpdateService.new(
-          project: project,
-          current_user: current_user,
-          params: {
-            escalation_status: {
-              status: status,
-              status_change_reason: " by changing the status of #{alert.to_reference(project)}"
-            }
-          }
-        ).execute(alert.issue)
-      end
-
-      def should_sync_to_incident?
-        alert.issue &&
-          alert.issue.supports_escalation? &&
-          alert.issue.escalation_status &&
-          alert.issue.escalation_status.status != alert.status
-      end
-
       def filter_duplicate
         # Only need to check if changing to a not-resolved status
         return if params[:status_event].blank? || params[:status_event] == :resolve
diff --git a/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb b/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb
index b5f105a6c89..d11492e062a 100644
--- a/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb
+++ b/app/services/incident_management/issuable_escalation_statuses/after_update_service.rb
@@ -6,7 +6,6 @@ module IncidentManagement
       def initialize(issuable, current_user, **params)
         @issuable = issuable
         @escalation_status = issuable.escalation_status
-        @alert = issuable.alert_management_alert
 
         super(project: issuable.project, current_user: current_user, params: params)
       end
@@ -19,26 +18,13 @@ module IncidentManagement
 
       private
 
-      attr_reader :issuable, :escalation_status, :alert
+      attr_reader :issuable, :escalation_status
 
       def after_update
-        sync_status_to_alert
         add_status_system_note
         add_timeline_event
       end
 
-      def sync_status_to_alert
-        return unless alert
-        return if alert.status == escalation_status.status
-
-        ::AlertManagement::Alerts::UpdateService.new(
-          alert,
-          current_user,
-          status: escalation_status.status_name,
-          status_change_reason: " by changing the incident status of #{issuable.to_reference(project)}"
-        ).execute
-      end
-
       def add_status_system_note
         return unless escalation_status.status_previously_changed?
 
diff --git a/app/services/incident_management/issuable_escalation_statuses/build_service.rb b/app/services/incident_management/issuable_escalation_statuses/build_service.rb
index 9ebcf72a0c9..b3c57da03cb 100644
--- a/app/services/incident_management/issuable_escalation_statuses/build_service.rb
+++ b/app/services/incident_management/issuable_escalation_statuses/build_service.rb
@@ -5,30 +5,17 @@ module IncidentManagement
     class BuildService < ::BaseProjectService
       def initialize(issue)
         @issue = issue
-        @alert = issue.alert_management_alert
 
         super(project: issue.project)
       end
 
       def execute
-        return issue.escalation_status if issue.escalation_status
-
-        issue.build_incident_management_issuable_escalation_status(alert_params)
+        issue.escalation_status || issue.build_incident_management_issuable_escalation_status
       end
 
       private
 
-      attr_reader :issue, :alert
-
-      def alert_params
-        return {} unless alert
-
-        {
-          status_event: alert.status_event_for(alert.status_name)
-        }
-      end
+      attr_reader :issue
     end
   end
 end
-
-IncidentManagement::IssuableEscalationStatuses::BuildService.prepend_mod
diff --git a/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb b/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb
index 1d0504a6e80..58777848151 100644
--- a/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb
+++ b/app/services/incident_management/issuable_escalation_statuses/prepare_update_service.rb
@@ -5,7 +5,7 @@ module IncidentManagement
     class PrepareUpdateService < ::BaseProjectService
       include Gitlab::Utils::StrongMemoize
 
-      SUPPORTED_PARAMS = %i[status status_change_reason].freeze
+      SUPPORTED_PARAMS = %i[status].freeze
 
       def initialize(issuable, current_user, params)
         @issuable = issuable
diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb
index 544b2170a02..acd6d45af7a 100644
--- a/app/services/issuable_base_service.rb
+++ b/app/services/issuable_base_service.rb
@@ -162,8 +162,6 @@ class IssuableBaseService < ::BaseProjectService
 
     return unless result.success? && result[:escalation_status].present?
 
-    @escalation_status_change_reason = result[:escalation_status].delete(:status_change_reason)
-
     params[:incident_management_issuable_escalation_status_attributes] = result[:escalation_status]
   end
 
diff --git a/app/services/issues/update_service.rb b/app/services/issues/update_service.rb
index d9210169005..afc61eed287 100644
--- a/app/services/issues/update_service.rb
+++ b/app/services/issues/update_service.rb
@@ -199,8 +199,7 @@ module Issues
 
       ::IncidentManagement::IssuableEscalationStatuses::AfterUpdateService.new(
         issue,
-        current_user,
-        status_change_reason: @escalation_status_change_reason # Defined in IssuableBaseService before save
+        current_user
       ).execute
     end
 
diff --git a/app/services/packages/cleanup/execute_policy_service.rb b/app/services/packages/cleanup/execute_policy_service.rb
new file mode 100644
index 00000000000..b432f6d0acb
--- /dev/null
+++ b/app/services/packages/cleanup/execute_policy_service.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Packages
+  module Cleanup
+    class ExecutePolicyService
+      include Gitlab::Utils::StrongMemoize
+
+      MAX_EXECUTION_TIME = 250.seconds
+
+      DUPLICATED_FILES_BATCH_SIZE = 10_000
+      MARK_PACKAGE_FILES_FOR_DESTRUCTION_SERVICE_BATCH_SIZE = 200
+
+      def initialize(policy)
+        @policy = policy
+        @counts = {
+          marked_package_files_total_count: 0,
+          unique_package_id_and_file_name_total_count: 0
+        }
+      end
+
+      def execute
+        cleanup_duplicated_files
+      end
+
+      private
+
+      def cleanup_duplicated_files
+        return if @policy.keep_n_duplicated_package_files_disabled?
+
+        result = installable_package_files.each_batch(of: DUPLICATED_FILES_BATCH_SIZE) do |package_files|
+          break :timeout if cleanup_duplicated_files_on(package_files) == :timeout
+        end
+
+        response_success(timeout: result == :timeout)
+      end
+
+      def cleanup_duplicated_files_on(package_files)
+        unique_package_id_and_file_name_from(package_files).each do |package_id, file_name|
+          result = remove_duplicated_files_for(package_id: package_id, file_name: file_name)
+          @counts[:marked_package_files_total_count] += result.payload[:marked_package_files_count]
+          @counts[:unique_package_id_and_file_name_total_count] += 1
+
+          break :timeout unless result.success?
+        end
+      end
+
+      def unique_package_id_and_file_name_from(package_files)
+        # This is a highly custom query for this service, that's why it's not in the model.
+        # rubocop: disable CodeReuse/ActiveRecord
+        package_files.group(:package_id, :file_name)
+          .having("COUNT(*) > #{@policy.keep_n_duplicated_package_files}")
+          .pluck(:package_id, :file_name)
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+
+      def remove_duplicated_files_for(package_id:, file_name:)
+        base = ::Packages::PackageFile.for_package_ids(package_id)
+                 .installable
+                 .with_file_name(file_name)
+        ids_to_keep = base.recent
+                        .limit(@policy.keep_n_duplicated_package_files)
+                        .pluck_primary_key
+
+        duplicated_package_files = base.id_not_in(ids_to_keep)
+        ::Packages::MarkPackageFilesForDestructionService.new(duplicated_package_files)
+          .execute(batch_deadline: batch_deadline, batch_size: MARK_PACKAGE_FILES_FOR_DESTRUCTION_SERVICE_BATCH_SIZE)
+      end
+
+      def project
+        @policy.project
+      end
+
+      def installable_package_files
+        ::Packages::PackageFile.installable
+          .for_package_ids(
+            ::Packages::Package.installable
+              .for_projects(project.id)
+          )
+      end
+
+      def batch_deadline
+        strong_memoize(:batch_deadline) do
+          MAX_EXECUTION_TIME.from_now
+        end
+      end
+
+      def response_success(timeout:)
+        ServiceResponse.success(
+          message: "Packages cleanup policy executed for project #{project.id}",
+          payload: {
+            timeout: timeout,
+            counts: @counts
+          }
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/packages/mark_package_files_for_destruction_service.rb b/app/services/packages/mark_package_files_for_destruction_service.rb
index 3672b44b409..e7fdd88843a 100644
--- a/app/services/packages/mark_package_files_for_destruction_service.rb
+++ b/app/services/packages/mark_package_files_for_destruction_service.rb
@@ -9,18 +9,41 @@ module Packages
       @package_files = package_files
     end
 
-    def execute
-      @package_files.each_batch(of: BATCH_SIZE) do |batched_package_files|
-        batched_package_files.update_all(status: :pending_destruction)
+    def execute(batch_deadline: nil, batch_size: BATCH_SIZE)
+      timeout = false
+      updates_count = 0
+      min_batch_size = [batch_size, BATCH_SIZE].min
+
+      @package_files.each_batch(of: min_batch_size) do |batched_package_files|
+        if batch_deadline && Time.zone.now > batch_deadline
+          timeout = true
+          break
+        end
+
+        updates_count += batched_package_files.update_all(status: :pending_destruction)
       end
 
-      service_response_success('Package files are now pending destruction')
+      payload = { marked_package_files_count: updates_count }
+
+      return response_error(payload) if timeout
+
+      response_success(payload)
     end
 
     private
 
-    def service_response_success(message)
-      ServiceResponse.success(message: message)
+    def response_success(payload)
+      ServiceResponse.success(
+        message: 'Package files are now pending destruction',
+        payload: payload
+      )
+    end
+
+    def response_error(payload)
+      ServiceResponse.error(
+        message: 'Timeout while marking package files as pending destruction',
+        payload: payload
+      )
     end
   end
 end