Add latest changes from gitlab-org/gitlab@master

11 files changed, 199 insertions, 92 deletions

diff --git a/.markdownlint.yml b/.markdownlint.yml
index ccf2ef02eee..4e852880d26 100644
--- a/.markdownlint.yml
+++ b/.markdownlint.yml
@@ -66,7 +66,6 @@
     "Grafana",
     "Gzip",
     "Helm",
-    "HipChat",
     "ID",
     "Ingress",
     "jasmine-jquery",
diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml
index 9a6e8451c97..ff736101b9e 100644
--- a/.rubocop_manual_todo.yml
+++ b/.rubocop_manual_todo.yml
@@ -1655,7 +1655,6 @@ Gitlab/NamespacedClass:
     - 'app/models/project_services/discord_service.rb'
     - 'app/models/project_services/drone_ci_service.rb'
     - 'app/models/project_services/hangouts_chat_service.rb'
-    - 'app/models/project_services/hipchat_service.rb'
     - 'app/models/project_services/issue_tracker_data.rb'
     - 'app/models/project_services/jenkins_service.rb'
     - 'app/models/project_services/jira_tracker_data.rb'
diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb
index 36edf646658..68c3b45c24e 100644
--- a/app/models/packages/package.rb
+++ b/app/models/packages/package.rb
@@ -8,6 +8,23 @@ class Packages::Package < ApplicationRecord
   DISPLAYABLE_STATUSES = [:default, :error].freeze
   INSTALLABLE_STATUSES = [:default].freeze
 
+  enum package_type: {
+    maven: 1,
+    npm: 2,
+    conan: 3,
+    nuget: 4,
+    pypi: 5,
+    composer: 6,
+    generic: 7,
+    golang: 8,
+    debian: 9,
+    rubygems: 10,
+    helm: 11,
+    terraform_module: 12
+  }
+
+  enum status: { default: 0, hidden: 1, processing: 2, error: 3 }
+
   belongs_to :project
   belongs_to :creator, class_name: 'User'
 
@@ -72,12 +89,6 @@ class Packages::Package < ApplicationRecord
     if: :debian_package?
   validate :forbidden_debian_changes, if: :debian?
 
-  enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5,
-                       composer: 6, generic: 7, golang: 8, debian: 9,
-                       rubygems: 10, helm: 11, terraform_module: 12 }
-
-  enum status: { default: 0, hidden: 1, processing: 2, error: 3 }
-
   scope :for_projects, ->(project_ids) { where(project_id: project_ids) }
   scope :with_name, ->(name) { where(name: name) }
   scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }
@@ -133,10 +144,28 @@ class Packages::Package < ApplicationRecord
   scope :order_type_desc, -> { reorder(package_type: :desc) }
   scope :order_project_name, -> { joins(:project).reorder('projects.name ASC') }
   scope :order_project_name_desc, -> { joins(:project).reorder('projects.name DESC') }
-  scope :order_project_path, -> { joins(:project).reorder('projects.path ASC, id ASC') }
-  scope :order_project_path_desc, -> { joins(:project).reorder('projects.path DESC, id DESC') }
   scope :order_by_package_file, -> { joins(:package_files).order('packages_package_files.created_at ASC') }
 
+  scope :order_project_path, -> do
+    if Feature.enabled?(:arel_package_scopes)
+      keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :asc)
+
+      joins(:project).reorder(keyset_order)
+    else
+      joins(:project).reorder('projects.path ASC, id ASC')
+    end
+  end
+
+  scope :order_project_path_desc, -> do
+    if Feature.enabled?(:arel_package_scopes)
+      keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :desc)
+
+      joins(:project).reorder(keyset_order)
+    else
+      joins(:project).reorder('projects.path DESC, id DESC')
+    end
+  end
+
   after_commit :update_composer_cache, on: :destroy, if: -> { composer? }
 
   def self.only_maven_packages_with_path(path, use_cte: false)
@@ -196,6 +225,32 @@ class Packages::Package < ApplicationRecord
     end
   end
 
+  def self.keyset_pagination_order(join_class:, column_name:, direction: :asc)
+    join_table = join_class.table_name
+    asc_order_expression = Gitlab::Database.nulls_last_order("#{join_table}.#{column_name}", :asc)
+    desc_order_expression = Gitlab::Database.nulls_first_order("#{join_table}.#{column_name}", :desc)
+    order_direction = direction == :asc ? asc_order_expression : desc_order_expression
+    reverse_order_direction = direction == :asc ? desc_order_expression : asc_order_expression
+    arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert
+
+    ::Gitlab::Pagination::Keyset::Order.build([
+      ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+        attribute_name: "#{join_table}_#{column_name}",
+        column_expression: join_class.arel_table[column_name],
+        order_expression: order_direction,
+        reversed_order_expression: reverse_order_direction,
+        order_direction: direction,
+        distinct: false,
+        add_to_projections: true
+      ),
+      ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+        attribute_name: 'id',
+        order_expression: arel_order_classes[direction].new(Packages::Package.arel_table[:id]),
+        add_to_projections: true
+      )
+    ])
+  end
+
   def versions
     project.packages
            .including_build_info
diff --git a/app/models/project.rb b/app/models/project.rb
index 31b527f405d..4bc16771323 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1406,9 +1406,9 @@ class Project < ApplicationRecord
   end
 
   def disabled_services
-    return %w[datadog hipchat] unless Feature.enabled?(:datadog_ci_integration, self)
+    return %w[datadog] unless Feature.enabled?(:datadog_ci_integration, self)
 
-    %w[hipchat]
+    []
   end
 
   def find_or_initialize_service(name)
diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb
deleted file mode 100644
index 71d8e7bfac4..00000000000
--- a/app/models/project_services/hipchat_service.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-# This service is scheduled for removal. All records must
-# be deleted before the class can be removed.
-# https://gitlab.com/gitlab-org/gitlab/-/issues/27954
-class HipchatService < Integration
-  before_save :prevent_save
-
-  def self.to_param
-    'hipchat'
-  end
-
-  def self.supported_events
-    []
-  end
-
-  def execute(data)
-    # We removed the hipchat gem due to https://gitlab.com/gitlab-org/gitlab/-/issues/325851#note_537143149
-    # HipChat is unusable anyway, so do nothing in this method
-  end
-
-  private
-
-  def prevent_save
-    errors.add(:base, _('HipChat endpoint is deprecated and should not be created or modified.'))
-
-    # Stops execution of callbacks and database operation while
-    # preserving expectations of #save (will not raise) & #save! (raises)
-    # https://guides.rubyonrails.org/active_record_callbacks.html#halting-execution
-    throw :abort # rubocop:disable Cop/BanCatchThrow
-  end
-end
diff --git a/config/feature_flags/development/arel_package_scopes.yml b/config/feature_flags/development/arel_package_scopes.yml
new file mode 100644
index 00000000000..7528e5f596d
--- /dev/null
+++ b/config/feature_flags/development/arel_package_scopes.yml
@@ -0,0 +1,8 @@
+---
+name: arel_package_scopes
+introduced_by_url: 
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/331306
+milestone: '13.12'
+type: development
+group: group::package
+default_enabled: false
diff --git a/doc/user/project/integrations/hipchat.md b/doc/user/project/integrations/hipchat.md
deleted file mode 100644
index 74366ab5ef0..00000000000
--- a/doc/user/project/integrations/hipchat.md
+++ /dev/null
@@ -1,9 +0,0 @@
----
-redirect_to: 'index.md'
-remove_date: '2021-06-30'
----
-
-This document was moved to [another location](index.md).
-
-<!-- This redirect file can be deleted after 2021-06-30. -->
-<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->
diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md
index 6c2acf260ff..14bf924886e 100644
--- a/doc/user/project/settings/index.md
+++ b/doc/user/project/settings/index.md
@@ -93,26 +93,82 @@ variables: # can be overriden by a developer's local .gitlab-ci.yml
 sast: # none of these attributes can be overriden by a developer's local .gitlab-ci.yml
   variables:
     FOO: sast
+  image: ruby:2.6
   stage: pre-compliance
+  rules:
+        - when: always
+  allow_failure: false
+  before_script:
+  - "# No before scripts."
   script:
   - echo "running $FOO"
+  after_script:
+  - "# No after scripts."
 
 sanity check:
+  image: ruby:2.6
   stage: pre-deploy-compliance
+  rules:
+        - when: always
+  allow_failure: false
+  before_script:
+  - "# No before scripts."  
   script:
   - echo "running $FOO"
+  after_script:
+  - "# No after scripts."
 
 
 audit trail:
+  image: ruby:2.6
   stage: post-compliance
+  rules:
+        - when: always
+  allow_failure: false
+  before_script:
+  - "# No before scripts."
   script:
   - echo "running $FOO"
+  after_script:
+  - "# No after scripts."
 
 include: # Execute individual project's configuration
   project: '$CI_PROJECT_PATH'
   file: '$CI_PROJECT_CONFIG_PATH'
 ```
 
+##### Ensure compliance jobs are always run
+
+Compliance pipelines use GitLab CI/CD to give you an incredible amount of flexibility
+for defining any sort of compliance jobs you like. Depending on your goals, these jobs
+can be configured to be:
+
+- Modified by users.
+- Non-modifiable.
+
+At a high-level, if a value in a compliance job:
+
+- Is set, it cannot be changed or overridden by project-level configurations.
+- Is not set, a project-level configuration may set.
+
+Either might be wanted or not depending on your use case.
+
+There are a few best practices for ensuring that these jobs are always run exactly
+as you define them and that downstream, project-level pipeline configurations
+cannot change them:
+
+- Add a `rules:when:always` block to each of your compliance jobs. This ensures they are
+  non-modifiable and are always run.
+- Explicitly set any variables the job references. This:
+  - Ensures that project-level pipeline configurations do not set them and alter their
+    behavior.
+  - Includes any jobs that drive the logic of your job.
+- Explicitly set the container image file to run the job in. This ensures that your script
+  steps execute in the correct environment.
+- Explicitly set any relevant GitLab pre-defined [job keywords](../../../ci/yaml/README.md#job-keywords).
+  This ensures that your job uses the settings you intend and that they are not overriden by
+  project-level pipelines.
+
 ### Sharing and permissions
 
 For your repository, you can set up features such as public access, repository features,
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 3d27d4b6bd4..f7747eb6252 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -16446,9 +16446,6 @@ msgstr ""
 msgid "HighlightBar|Time to SLA:"
 msgstr ""
 
-msgid "HipChat endpoint is deprecated and should not be created or modified."
-msgstr ""
-
 msgid "History"
 msgstr ""
 
diff --git a/spec/models/packages/package_spec.rb b/spec/models/packages/package_spec.rb
index 52ef61e3d44..b8bf4f9683c 100644
--- a/spec/models/packages/package_spec.rb
+++ b/spec/models/packages/package_spec.rb
@@ -729,6 +729,49 @@ RSpec.describe Packages::Package, type: :model do
     end
   end
 
+  context 'sorting' do
+    let_it_be(:project) { create(:project, name: 'aaa' ) }
+    let_it_be(:project2) { create(:project, name: 'bbb' ) }
+    let_it_be(:package1) { create(:package, project: project ) }
+    let_it_be(:package2) { create(:package, project: project2 ) }
+    let_it_be(:package3) { create(:package, project: project2 ) }
+    let_it_be(:package4) { create(:package, project: project ) }
+
+    it 'orders packages by their projects name ascending' do
+      expect(Packages::Package.order_project_name).to eq([package1, package4, package2, package3])
+    end
+
+    it 'orders packages by their projects name descending' do
+      expect(Packages::Package.order_project_name_desc).to eq([package2, package3, package1, package4])
+    end
+
+    shared_examples 'order_project_path scope' do
+      it 'orders packages by their projects path asc, then package id asc' do
+        expect(Packages::Package.order_project_path).to eq([package1, package4, package2, package3])
+      end
+    end
+
+    shared_examples 'order_project_path_desc scope' do
+      it 'orders packages by their projects path desc, then package id desc' do
+        expect(Packages::Package.order_project_path_desc).to eq([package3, package2, package4, package1])
+      end
+    end
+
+    context 'with arel scope feature flag enabled' do
+      it_behaves_like 'order_project_path scope'
+      it_behaves_like 'order_project_path_desc scope'
+    end
+
+    context 'with feature flag disabled' do
+      before do
+        stub_feature_flags(arel_package_scopes: false)
+      end
+
+      it_behaves_like 'order_project_path scope'
+      it_behaves_like 'order_project_path_desc scope'
+    end
+  end
+
   describe '.order_by_package_file' do
     let_it_be(:project) { create(:project) }
     let_it_be(:package1) { create(:maven_package, project: project) }
@@ -743,6 +786,33 @@ RSpec.describe Packages::Package, type: :model do
     end
   end
 
+  describe '.keyset_pagination_order' do
+    let(:join_class) { nil }
+    let(:column_name) { nil }
+    let(:direction) { nil }
+
+    subject { described_class.keyset_pagination_order(join_class: join_class, column_name: column_name, direction: direction) }
+
+    it { expect { subject }.to raise_error(NoMethodError) }
+
+    context 'with valid params' do
+      let(:join_class) { Project }
+      let(:column_name) { :name }
+
+      context 'ascending direction' do
+        let(:direction) { :asc }
+
+        it { is_expected.to eq('projects.name asc NULLS LAST, "packages_packages"."id" ASC') }
+      end
+
+      context 'descending direction' do
+        let(:direction) { :desc }
+
+        it { is_expected.to eq('projects.name desc NULLS FIRST, "packages_packages"."id" DESC') }
+      end
+    end
+  end
+
   describe '#versions' do
     let_it_be(:project) { create(:project) }
     let_it_be(:package) { create(:maven_package, project: project) }
diff --git a/spec/models/project_services/hipchat_service_spec.rb b/spec/models/project_services/hipchat_service_spec.rb
deleted file mode 100644
index 42368c31ba0..00000000000
--- a/spec/models/project_services/hipchat_service_spec.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-# HipchatService is partially removed and it will be remove completely
-# after the deletion of all the database records.
-# https://gitlab.com/gitlab-org/gitlab/-/issues/27954
-RSpec.describe HipchatService do
-  let_it_be(:project) { create(:project) }
-
-  subject(:service) { described_class.new(project: project) }
-
-  it { is_expected.to be_valid }
-
-  describe '#to_param' do
-    subject { service.to_param }
-
-    it { is_expected.to eq('hipchat') }
-  end
-
-  describe '#supported_events' do
-    subject { service.supported_events }
-
-    it { is_expected.to be_empty }
-  end
-
-  describe '#save' do
-    it 'prevents records from being created or updated' do
-      expect(service.save).to be_falsey
-
-      expect(service.errors.full_messages).to include(
-        'HipChat endpoint is deprecated and should not be created or modified.'
-      )
-    end
-  end
-end