Add latest changes from gitlab-org/gitlab@master

19 files changed, 154 insertions, 49 deletions

diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 07684334b57..a9d42925fc7 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-996add2f4e011cec8e9317912978cf1fa59e66c1
+4ea88e921af65ba0577c40f8b54830c97adaa56c
diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue
index dafe1cfe2a9..1fb4bd26533 100644
--- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue
+++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue
@@ -18,8 +18,15 @@ export default {
     GlSprintf,
     GlButton,
   },
-  inject: ['fullPath', 'iid'],
   props: {
+    iid: {
+      type: String,
+      required: true,
+    },
+    fullPath: {
+      type: String,
+      required: true,
+    },
     confidential: {
       required: true,
       type: Boolean,
diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue
index ec5f07f9785..372368707af 100644
--- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue
+++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue
@@ -27,8 +27,20 @@ export default {
     SidebarConfidentialityContent,
     SidebarConfidentialityForm,
   },
-  inject: ['fullPath', 'iid'],
+  inject: {
+    isClassicSidebar: {
+      default: false,
+    },
+  },
   props: {
+    iid: {
+      type: String,
+      required: true,
+    },
+    fullPath: {
+      type: String,
+      required: true,
+    },
     issuableType: {
       required: true,
       type: String,
@@ -126,6 +138,7 @@ export default {
           v-if="!isLoading"
           :confidential="confidential"
           :issuable-type="issuableType"
+          :class="{ 'gl-mt-3': !isClassicSidebar }"
           @expandSidebar="expandSidebar"
         />
       </div>
@@ -133,6 +146,8 @@ export default {
     <template #default>
       <sidebar-confidentiality-content :confidential="confidential" :issuable-type="issuableType" />
       <sidebar-confidentiality-form
+        :iid="iid"
+        :full-path="fullPath"
         :confidential="confidential"
         :issuable-type="issuableType"
         @closeForm="closeForm"
diff --git a/app/assets/javascripts/sidebar/mount_sidebar.js b/app/assets/javascripts/sidebar/mount_sidebar.js
index f5c8ed6bd61..643a57bf6a8 100644
--- a/app/assets/javascripts/sidebar/mount_sidebar.js
+++ b/app/assets/javascripts/sidebar/mount_sidebar.js
@@ -190,14 +190,14 @@ function mountConfidentialComponent() {
       SidebarConfidentialityWidget,
     },
     provide: {
-      iid: String(iid),
-      fullPath,
       canUpdate: initialData.is_editable,
     },
 
     render: (createElement) =>
       createElement('sidebar-confidentiality-widget', {
         props: {
+          iid: String(iid),
+          fullPath,
           issuableType:
             isInIssuePage() || isInIncidentPage() || isInDesignPage()
               ? IssuableType.Issue
diff --git a/app/assets/stylesheets/page_bundles/boards.scss b/app/assets/stylesheets/page_bundles/boards.scss
index e8e3a8f6591..a00a71b07e7 100644
--- a/app/assets/stylesheets/page_bundles/boards.scss
+++ b/app/assets/stylesheets/page_bundles/boards.scss
@@ -468,6 +468,12 @@
   }
 }
 
+.boards-sidebar {
+  .sidebar-collapsed-icon {
+    display: none;
+  }
+}
+
 .board-header-collapsed-info-icon:hover {
   color: var(--gray-900, $gray-900);
 }
diff --git a/app/helpers/labels_helper.rb b/app/helpers/labels_helper.rb
index 6cc9959dcb9..cfc4075100b 100644
--- a/app/helpers/labels_helper.rb
+++ b/app/helpers/labels_helper.rb
@@ -164,8 +164,8 @@ module LabelsHelper
   end
 
   def label_subscription_status(label, project)
-    return 'group-level' if label.lazy_subscribed?(current_user)
-    return 'project-level' if label.lazy_subscribed?(current_user, project)
+    return 'group-level' if label.subscribed?(current_user)
+    return 'project-level' if label.subscribed?(current_user, project)
 
     'unsubscribed'
   end
@@ -181,7 +181,7 @@ module LabelsHelper
   end
 
   def label_subscription_toggle_button_text(label, project = nil)
-    label.lazy_subscribed?(current_user, project) ? 'Unsubscribe' : 'Subscribe'
+    label.subscribed?(current_user, project) ? 'Unsubscribe' : 'Subscribe'
   end
 
   def create_label_title(subject)
diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb
index 10bfea0965e..5a10ea7a248 100644
--- a/app/models/concerns/subscribable.rb
+++ b/app/models/concerns/subscribable.rb
@@ -17,16 +17,6 @@ module Subscribable
   def subscribed?(user, project = nil)
     return false unless user
 
-    if (subscription = subscriptions.find_by(user: user, project: project))
-      subscription.subscribed
-    else
-      subscribed_without_subscriptions?(user, project)
-    end
-  end
-
-  def lazy_subscribed?(user, project = nil)
-    return false unless user
-
     if (subscription = lazy_subscription(user, project)&.itself)
       subscription.subscribed
     else
@@ -75,8 +65,10 @@ module Subscribable
   def toggle_subscription(user, project = nil)
     unsubscribe_from_other_levels(user, project)
 
+    new_value = !subscribed?(user, project)
+
     find_or_initialize_subscription(user, project)
-      .update(subscribed: !subscribed?(user, project))
+      .update(subscribed: new_value)
   end
 
   def subscribe(user, project = nil)
@@ -117,6 +109,8 @@ module Subscribable
   end
 
   def find_or_initialize_subscription(user, project)
+    BatchLoader::Executor.clear_current
+
     subscriptions
       .find_or_initialize_by(user_id: user.id, project_id: project.try(:id))
   end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 907329f6991..af78466e6a9 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -115,6 +115,7 @@ class Issue < ApplicationRecord
 
   scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) }
   scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) }
+  scope :preload_awardable, -> { preload(:award_emoji) }
   scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) }
   scope :with_alert_management_alerts, -> { joins(:alert_management_alert) }
   scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) }
@@ -343,6 +344,8 @@ class Issue < ApplicationRecord
                        .preload(preload)
                        .reorder('issue_link_id')
 
+    related_issues = yield related_issues if block_given?
+
     cross_project_filter = -> (issues) { issues.where(project: project) }
     Ability.issues_readable_by_user(related_issues,
       current_user,
diff --git a/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml b/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml
new file mode 100644
index 00000000000..d16ef0fb7ab
--- /dev/null
+++ b/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml
@@ -0,0 +1,5 @@
+---
+title: Reduce SQL requests number for issue links
+merge_request: 57602
+author:
+type: performance
diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md
index 88d86f1213b..4d2ef141622 100644
--- a/doc/administration/pages/index.md
+++ b/doc/administration/pages/index.md
@@ -938,6 +938,67 @@ In installations from source:
 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source)
    for the changes to take effect.
 
+## ZIP storage
+
+In GitLab 14.0 the underlaying storage format of GitLab Pages is changing from
+files stored directly in disk to a single ZIP archive per project.
+
+These ZIP archives can be stored either locally on disk storage or on the [object storage](#using-object-storage) if it is configured.
+
+[Starting from GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/245308) ZIP archives are stored every time pages site is updated.
+
+### Migrate legacy storage to ZIP storage
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59003) in GitLab 13.11.
+
+GitLab will [try to automatically migrate](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578) the old storage format to the new ZIP-based one when you upgrade to GitLab 13.11 or further.
+However, some projects may fail to be migrated for different reasons.
+To verify that all projects have been migrated successfully, you can manually run the migration:
+
+```shell
+gitlab-rake gitlab:pages:migrate_legacy_storage
+```
+
+It's safe to interrupt this task and run it multiple times.
+
+There are two most common problems this task can report:
+
+- `Missing public directory` error:
+
+  ```txt
+  E, [2021-04-09T13:11:52.534768 #911919] ERROR -- : project_id: 1 /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test failed to be migrated in 0.07 seconds: Archive not created. Missing public directory in /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test
+  ```
+
+  In this case, you should verify that these projects don't have pages deployed, and re-run the migration with an additional flag to mark those projects as not deployed with GitLab Pages:
+
+  ```shell
+  sudo PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED=true gitlab-rake gitlab:pages:migrate_legacy_storage
+  ```
+
+- File `is invalid` error:
+
+  ```txt
+  E, [2021-04-09T14:43:05.821767 #923322] ERROR -- : project_id: 1 /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test failed to be migrated: /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test/public/link is invalid, input_dir: /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test
+  ```
+
+  This error indicates invalid files on disk storage, most commonly symlinks leading outside of the `public` directory.
+  You can manually remove these files, or just ignore them during migration:
+
+  ```shell
+  sudo PAGES_MIGRATION_IGNORE_INVALID_ENTRIES=true gitlab-rake gitlab:pages:migrate_legacy_storage
+  ```
+
+### Rolling back ZIP migration
+
+If you find that migrated data is invalid, you can remove all migrated data by running:
+
+```shell
+sudo gitlab-rake gitlab:pages:clean_migrated_zip_storage
+```
+
+This will not remove any data from the legacy disk storage and the GitLab Pages daemon will automatically fallback
+to using that.
+
 ## Backup
 
 GitLab Pages are part of the [regular backup](../../raketasks/backup_restore.md), so there is no separate backup to configure.
diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md
index 1cd1c146b1e..97af1fe8d3c 100644
--- a/doc/administration/reference_architectures/10k_users.md
+++ b/doc/administration/reference_architectures/10k_users.md
@@ -22,10 +22,10 @@ full list of reference architectures, see
 | PostgreSQL*                                | 3           | 8 vCPU, 30 GB memory    | `n1-standard-8`  | `m5.2xlarge` | `D8s v3`  |
 | PgBouncer*                                 | 3           | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   | `c5.large`   | `F2s v2`  |
 | Internal load balancing node               | 1           | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   | `c5.large`   | `F2s v2`  |
-| Redis - Cache*                             | 3           | 4 vCPU, 15 GB memory    | `n1-standard-4`  | `m5.xlarge`  | `D4s v3`  |
-| Redis - Queues / Shared State*             | 3           | 4 vCPU, 15 GB memory    | `n1-standard-4`  | `m5.xlarge`  | `D4s v3`  |
-| Redis Sentinel - Cache*                    | 3           | 1 vCPU, 1.7 GB memory   | `g1-small`       | `t3.small`   | `B1MS`    |
-| Redis Sentinel - Queues / Shared State*    | 3           | 1 vCPU, 1.7 GB memory   | `g1-small`       | `t3.small`   | `B1MS`    |
+| Redis - Cache**                            | 3           | 4 vCPU, 15 GB memory    | `n1-standard-4`  | `m5.xlarge`  | `D4s v3`  |
+| Redis - Queues / Shared State**            | 3           | 4 vCPU, 15 GB memory    | `n1-standard-4`  | `m5.xlarge`  | `D4s v3`  |
+| Redis Sentinel - Cache**                   | 3           | 1 vCPU, 1.7 GB memory   | `g1-small`       | `t3.small`   | `B1MS`    |
+| Redis Sentinel - Queues / Shared State**   | 3           | 1 vCPU, 1.7 GB memory   | `g1-small`       | `t3.small`   | `B1MS`    |
 | Gitaly                                     | 3           | 16 vCPU, 60 GB memory   | `n1-standard-16` | `m5.4xlarge` | `D16s v3` |
 | Praefect                                   | 3           | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   | `c5.large`   | `F2s v2`  |
 | Praefect PostgreSQL*                       | 1+          | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   | `c5.large`   | `F2s v2`  |
@@ -37,7 +37,9 @@ full list of reference architectures, see
 
 NOTE:
 Components marked with * can be optionally run on reputable
-third party external PaaS solutions such as Google Cloud SQL or Memorystore.
+third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work.
+Components marked with ** can be optionally run on reputable
+third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work.
 
 ```plantuml
 @startuml 10k
@@ -2401,10 +2403,10 @@ services where applicable):
 | PostgreSQL*                                | 3     | 8 vCPU, 30 GB memory    | `n1-standard-8`  |
 | PgBouncer*                                 | 3     | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   |
 | Internal load balancing node               | 1     | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   |
-| Redis - Cache*                             | 3     | 4 vCPU, 15 GB memory    | `n1-standard-4`  |
-| Redis - Queues / Shared State*             | 3     | 4 vCPU, 15 GB memory    | `n1-standard-4`  |
-| Redis Sentinel - Cache*                    | 3     | 1 vCPU, 1.7 GB memory   | `g1-small`       |
-| Redis Sentinel - Queues / Shared State*    | 3     | 1 vCPU, 1.7 GB memory   | `g1-small`       |
+| Redis - Cache**                            | 3     | 4 vCPU, 15 GB memory    | `n1-standard-4`  |
+| Redis - Queues / Shared State**            | 3     | 4 vCPU, 15 GB memory    | `n1-standard-4`  |
+| Redis Sentinel - Cache**                   | 3     | 1 vCPU, 1.7 GB memory   | `g1-small`       |
+| Redis Sentinel - Queues / Shared State**   | 3     | 1 vCPU, 1.7 GB memory   | `g1-small`       |
 | Gitaly                                     | 3     | 16 vCPU, 60 GB memory   | `n1-standard-16` |
 | Praefect                                   | 3     | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   |
 | Praefect PostgreSQL*                       | 1+    | 2 vCPU, 1.8 GB memory   | `n1-highcpu-2`   |
@@ -2412,7 +2414,9 @@ services where applicable):
 
 NOTE:
 Components marked with * can be optionally run on reputable
-third party external PaaS solutions such as Google Cloud SQL or Memorystore.
+third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work.
+Components marked with ** can be optionally run on reputable
+third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work.
 
 ```plantuml
 @startuml 10k
diff --git a/doc/api/README.md b/doc/api/README.md
index b310ca0fd7d..1a914fb1dbe 100644
--- a/doc/api/README.md
+++ b/doc/api/README.md
@@ -760,7 +760,7 @@ The correct encoding for the query parameter would be:
 ## Clients
 
 There are many unofficial GitLab API Clients for most of the popular programming
-languages. For a complete list, visit the [GitLab website](https://about.gitlab.com/partners/#api-clients).
+languages. For a complete list, visit the [GitLab website](https://about.gitlab.com/partners/technology-partners/#api-clients).
 
 ## Rate limits
 
diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md
index 1e536b0cbec..6e38534b044 100644
--- a/doc/user/gitlab_com/index.md
+++ b/doc/user/gitlab_com/index.md
@@ -113,7 +113,7 @@ or over the repository size limit, you can [reduce your repository size with Git
 
 | Setting                       | GitLab.com  | Default       |
 | -----------                   | ----------- | ------------- |
-| [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md) | 10 GB       | Unlimited     |
+| [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md#repository-size-limit) | 10 GB       | Unlimited     |
 | Maximum import size           | 5 GB        | Unlimited ([Modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to unlimited in GitLab 13.8.    |
 
 NOTE:
diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb
index e938dbbae87..1cd5bde224b 100644
--- a/lib/api/issue_links.rb
+++ b/lib/api/issue_links.rb
@@ -18,7 +18,10 @@ module API
       end
       get ':id/issues/:issue_iid/links' do
         source_issue = find_project_issue(params[:issue_iid])
-        related_issues = source_issue.related_issues(current_user)
+        related_issues = source_issue.related_issues(current_user) do |issues|
+          issues.with_api_entity_associations.preload_awardable
+        end
+        related_issues.each { |issue| issue.lazy_subscription(current_user, user_project) } # preload subscriptions
 
         present related_issues,
                 with: Entities::RelatedIssue,
diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb
index e348149bb46..f970457fea7 100644
--- a/scripts/gitaly_test.rb
+++ b/scripts/gitaly_test.rb
@@ -52,8 +52,7 @@ module GitalyTest
       'RUBYOPT' => nil,
 
       # Git hooks can't run during tests as the internal API is not running.
-      'GITALY_TESTING_NO_GIT_HOOKS' => "1",
-      'GITALY_TESTING_ENABLE_ALL_FEATURE_FLAGS' => "true"
+      'GITALY_TESTING_NO_GIT_HOOKS' => "1"
     }
 
     env_hash
diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js
index d5e6310ed38..28a19fb9df6 100644
--- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js
+++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js
@@ -20,11 +20,9 @@ describe('Sidebar Confidentiality Form', () => {
     mutate = jest.fn().mockResolvedValue('Success'),
   } = {}) => {
     wrapper = shallowMount(SidebarConfidentialityForm, {
-      provide: {
+      propsData: {
         fullPath: 'group/project',
         iid: '1',
-      },
-      propsData: {
         confidential: false,
         issuableType: 'issue',
         ...props,
diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js
index 20a5be9b518..707215d0739 100644
--- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js
+++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js
@@ -35,11 +35,11 @@ describe('Sidebar Confidentiality Widget', () => {
       localVue,
       apolloProvider: fakeApollo,
       provide: {
-        fullPath: 'group/project',
-        iid: '1',
         canUpdate: true,
       },
       propsData: {
+        fullPath: 'group/project',
+        iid: '1',
         issuableType: 'issue',
       },
       stubs: {
diff --git a/spec/models/concerns/subscribable_spec.rb b/spec/models/concerns/subscribable_spec.rb
index e808281b862..a60a0a5e26d 100644
--- a/spec/models/concerns/subscribable_spec.rb
+++ b/spec/models/concerns/subscribable_spec.rb
@@ -194,10 +194,6 @@ RSpec.describe Subscribable, 'Subscribable' do
     end
   end
 
-  describe '#lazy_subscribed?' do
-    it_behaves_like 'returns expected values', :lazy_subscribed?
-  end
-
   describe '#lazy_subscription' do
     let(:labels) { create_list(:group_label, 5) }
 
diff --git a/spec/requests/api/issue_links_spec.rb b/spec/requests/api/issue_links_spec.rb
index b64d395f25f..45583f5c7dc 100644
--- a/spec/requests/api/issue_links_spec.rb
+++ b/spec/requests/api/issue_links_spec.rb
@@ -12,26 +12,40 @@ RSpec.describe API::IssueLinks do
   end
 
   describe 'GET /links' do
+    def perform_request(user = nil, params = {})
+      get api("/projects/#{project.id}/issues/#{issue.iid}/links", user), params: params
+    end
+
     context 'when unauthenticated' do
       it 'returns 401' do
-        get api("/projects/#{project.id}/issues/#{issue.iid}/links")
+        perform_request
 
         expect(response).to have_gitlab_http_status(:unauthorized)
       end
     end
 
     context 'when authenticated' do
-      it 'returns related issues' do
-        target_issue = create(:issue, project: project)
-        create(:issue_link, source: issue, target: target_issue)
+      let_it_be(:issue_link1) { create(:issue_link, source: issue, target: create(:issue, project: project)) }
+      let_it_be(:issue_link2) { create(:issue_link, source: issue, target: create(:issue, project: project)) }
 
-        get api("/projects/#{project.id}/issues/#{issue.iid}/links", user)
+      it 'returns related issues' do
+        perform_request(user)
 
         expect(response).to have_gitlab_http_status(:ok)
         expect(json_response).to be_an Array
-        expect(json_response.length).to eq(1)
+        expect(json_response.length).to eq(2)
         expect(response).to match_response_schema('public_api/v4/issue_links')
       end
+
+      it 'returns multiple links without N + 1' do
+        perform_request(user)
+
+        control_count = ActiveRecord::QueryRecorder.new { perform_request(user) }.count
+
+        create(:issue_link, source: issue, target: create(:issue, project: project))
+
+        expect { perform_request(user) }.not_to exceed_query_limit(control_count)
+      end
     end
   end