diff options
19 files changed, 154 insertions, 49 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 07684334b57..a9d42925fc7 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -996add2f4e011cec8e9317912978cf1fa59e66c1 +4ea88e921af65ba0577c40f8b54830c97adaa56c diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue index dafe1cfe2a9..1fb4bd26533 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_form.vue @@ -18,8 +18,15 @@ export default { GlSprintf, GlButton, }, - inject: ['fullPath', 'iid'], props: { + iid: { + type: String, + required: true, + }, + fullPath: { + type: String, + required: true, + }, confidential: { required: true, type: Boolean, diff --git a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue index ec5f07f9785..372368707af 100644 --- a/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue +++ b/app/assets/javascripts/sidebar/components/confidential/sidebar_confidentiality_widget.vue @@ -27,8 +27,20 @@ export default { SidebarConfidentialityContent, SidebarConfidentialityForm, }, - inject: ['fullPath', 'iid'], + inject: { + isClassicSidebar: { + default: false, + }, + }, props: { + iid: { + type: String, + required: true, + }, + fullPath: { + type: String, + required: true, + }, issuableType: { required: true, type: String, @@ -126,6 +138,7 @@ export default { v-if="!isLoading" :confidential="confidential" :issuable-type="issuableType" + :class="{ 'gl-mt-3': !isClassicSidebar }" @expandSidebar="expandSidebar" /> </div> @@ -133,6 +146,8 @@ export default { <template #default> <sidebar-confidentiality-content :confidential="confidential" :issuable-type="issuableType" /> <sidebar-confidentiality-form + :iid="iid" + :full-path="fullPath" :confidential="confidential" :issuable-type="issuableType" @closeForm="closeForm" diff --git a/app/assets/javascripts/sidebar/mount_sidebar.js b/app/assets/javascripts/sidebar/mount_sidebar.js index f5c8ed6bd61..643a57bf6a8 100644 --- a/app/assets/javascripts/sidebar/mount_sidebar.js +++ b/app/assets/javascripts/sidebar/mount_sidebar.js @@ -190,14 +190,14 @@ function mountConfidentialComponent() { SidebarConfidentialityWidget, }, provide: { - iid: String(iid), - fullPath, canUpdate: initialData.is_editable, }, render: (createElement) => createElement('sidebar-confidentiality-widget', { props: { + iid: String(iid), + fullPath, issuableType: isInIssuePage() || isInIncidentPage() || isInDesignPage() ? IssuableType.Issue diff --git a/app/assets/stylesheets/page_bundles/boards.scss b/app/assets/stylesheets/page_bundles/boards.scss index e8e3a8f6591..a00a71b07e7 100644 --- a/app/assets/stylesheets/page_bundles/boards.scss +++ b/app/assets/stylesheets/page_bundles/boards.scss @@ -468,6 +468,12 @@ } } +.boards-sidebar { + .sidebar-collapsed-icon { + display: none; + } +} + .board-header-collapsed-info-icon:hover { color: var(--gray-900, $gray-900); } diff --git a/app/helpers/labels_helper.rb b/app/helpers/labels_helper.rb index 6cc9959dcb9..cfc4075100b 100644 --- a/app/helpers/labels_helper.rb +++ b/app/helpers/labels_helper.rb @@ -164,8 +164,8 @@ module LabelsHelper end def label_subscription_status(label, project) - return 'group-level' if label.lazy_subscribed?(current_user) - return 'project-level' if label.lazy_subscribed?(current_user, project) + return 'group-level' if label.subscribed?(current_user) + return 'project-level' if label.subscribed?(current_user, project) 'unsubscribed' end @@ -181,7 +181,7 @@ module LabelsHelper end def label_subscription_toggle_button_text(label, project = nil) - label.lazy_subscribed?(current_user, project) ? 'Unsubscribe' : 'Subscribe' + label.subscribed?(current_user, project) ? 'Unsubscribe' : 'Subscribe' end def create_label_title(subject) diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 10bfea0965e..5a10ea7a248 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -17,16 +17,6 @@ module Subscribable def subscribed?(user, project = nil) return false unless user - if (subscription = subscriptions.find_by(user: user, project: project)) - subscription.subscribed - else - subscribed_without_subscriptions?(user, project) - end - end - - def lazy_subscribed?(user, project = nil) - return false unless user - if (subscription = lazy_subscription(user, project)&.itself) subscription.subscribed else @@ -75,8 +65,10 @@ module Subscribable def toggle_subscription(user, project = nil) unsubscribe_from_other_levels(user, project) + new_value = !subscribed?(user, project) + find_or_initialize_subscription(user, project) - .update(subscribed: !subscribed?(user, project)) + .update(subscribed: new_value) end def subscribe(user, project = nil) @@ -117,6 +109,8 @@ module Subscribable end def find_or_initialize_subscription(user, project) + BatchLoader::Executor.clear_current + subscriptions .find_or_initialize_by(user_id: user.id, project_id: project.try(:id)) end diff --git a/app/models/issue.rb b/app/models/issue.rb index 907329f6991..af78466e6a9 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -115,6 +115,7 @@ class Issue < ApplicationRecord scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } + scope :preload_awardable, -> { preload(:award_emoji) } scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } @@ -343,6 +344,8 @@ class Issue < ApplicationRecord .preload(preload) .reorder('issue_link_id') + related_issues = yield related_issues if block_given? + cross_project_filter = -> (issues) { issues.where(project: project) } Ability.issues_readable_by_user(related_issues, current_user, diff --git a/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml b/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml new file mode 100644 index 00000000000..d16ef0fb7ab --- /dev/null +++ b/changelogs/unreleased/id-reduce-sql-requests-for-issue-links.yml @@ -0,0 +1,5 @@ +--- +title: Reduce SQL requests number for issue links +merge_request: 57602 +author: +type: performance diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 88d86f1213b..4d2ef141622 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -938,6 +938,67 @@ In installations from source: 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. +## ZIP storage + +In GitLab 14.0 the underlaying storage format of GitLab Pages is changing from +files stored directly in disk to a single ZIP archive per project. + +These ZIP archives can be stored either locally on disk storage or on the [object storage](#using-object-storage) if it is configured. + +[Starting from GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/245308) ZIP archives are stored every time pages site is updated. + +### Migrate legacy storage to ZIP storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59003) in GitLab 13.11. + +GitLab will [try to automatically migrate](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578) the old storage format to the new ZIP-based one when you upgrade to GitLab 13.11 or further. +However, some projects may fail to be migrated for different reasons. +To verify that all projects have been migrated successfully, you can manually run the migration: + +```shell +gitlab-rake gitlab:pages:migrate_legacy_storage +``` + +It's safe to interrupt this task and run it multiple times. + +There are two most common problems this task can report: + +- `Missing public directory` error: + + ```txt + E, [2021-04-09T13:11:52.534768 #911919] ERROR -- : project_id: 1 /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test failed to be migrated in 0.07 seconds: Archive not created. Missing public directory in /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test + ``` + + In this case, you should verify that these projects don't have pages deployed, and re-run the migration with an additional flag to mark those projects as not deployed with GitLab Pages: + + ```shell + sudo PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED=true gitlab-rake gitlab:pages:migrate_legacy_storage + ``` + +- File `is invalid` error: + + ```txt + E, [2021-04-09T14:43:05.821767 #923322] ERROR -- : project_id: 1 /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test failed to be migrated: /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test/public/link is invalid, input_dir: /home/vlad/gdk/gitlab/shared/pages/gitlab-org/gitlab-test + ``` + + This error indicates invalid files on disk storage, most commonly symlinks leading outside of the `public` directory. + You can manually remove these files, or just ignore them during migration: + + ```shell + sudo PAGES_MIGRATION_IGNORE_INVALID_ENTRIES=true gitlab-rake gitlab:pages:migrate_legacy_storage + ``` + +### Rolling back ZIP migration + +If you find that migrated data is invalid, you can remove all migrated data by running: + +```shell +sudo gitlab-rake gitlab:pages:clean_migrated_zip_storage +``` + +This will not remove any data from the legacy disk storage and the GitLab Pages daemon will automatically fallback +to using that. + ## Backup GitLab Pages are part of the [regular backup](../../raketasks/backup_restore.md), so there is no separate backup to configure. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 1cd1c146b1e..97af1fe8d3c 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -22,10 +22,10 @@ full list of reference architectures, see | PostgreSQL* | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | | PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis - Cache* | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis - Queues / Shared State* | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis Sentinel - Cache* | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | `t3.small` | `B1MS` | -| Redis Sentinel - Queues / Shared State* | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | `t3.small` | `B1MS` | +| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis Sentinel - Cache** | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | `t3.small` | `B1MS` | +| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | `t3.small` | `B1MS` | | Gitaly | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | @@ -37,7 +37,9 @@ full list of reference architectures, see NOTE: Components marked with * can be optionally run on reputable -third party external PaaS solutions such as Google Cloud SQL or Memorystore. +third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. +Components marked with ** can be optionally run on reputable +third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. ```plantuml @startuml 10k @@ -2401,10 +2403,10 @@ services where applicable): | PostgreSQL* | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | | PgBouncer* | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | | Internal load balancing node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | -| Redis - Cache* | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | -| Redis - Queues / Shared State* | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | -| Redis Sentinel - Cache* | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | -| Redis Sentinel - Queues / Shared State* | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | +| Redis - Cache** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis - Queues / Shared State** | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | +| Redis Sentinel - Cache** | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | +| Redis Sentinel - Queues / Shared State** | 3 | 1 vCPU, 1.7 GB memory | `g1-small` | | Gitaly | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | | Praefect | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | | Praefect PostgreSQL* | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | @@ -2412,7 +2414,9 @@ services where applicable): NOTE: Components marked with * can be optionally run on reputable -third party external PaaS solutions such as Google Cloud SQL or Memorystore. +third party external PaaS PostgreSQL solutions. Google Cloud SQL and AWS RDS are known to work. +Components marked with ** can be optionally run on reputable +third party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. ```plantuml @startuml 10k diff --git a/doc/api/README.md b/doc/api/README.md index b310ca0fd7d..1a914fb1dbe 100644 --- a/doc/api/README.md +++ b/doc/api/README.md @@ -760,7 +760,7 @@ The correct encoding for the query parameter would be: ## Clients There are many unofficial GitLab API Clients for most of the popular programming -languages. For a complete list, visit the [GitLab website](https://about.gitlab.com/partners/#api-clients). +languages. For a complete list, visit the [GitLab website](https://about.gitlab.com/partners/technology-partners/#api-clients). ## Rate limits diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 1e536b0cbec..6e38534b044 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -113,7 +113,7 @@ or over the repository size limit, you can [reduce your repository size with Git | Setting | GitLab.com | Default | | ----------- | ----------- | ------------- | -| [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md) | 10 GB | Unlimited | +| [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md#repository-size-limit) | 10 GB | Unlimited | | Maximum import size | 5 GB | Unlimited ([Modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to unlimited in GitLab 13.8. | NOTE: diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index e938dbbae87..1cd5bde224b 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -18,7 +18,10 @@ module API end get ':id/issues/:issue_iid/links' do source_issue = find_project_issue(params[:issue_iid]) - related_issues = source_issue.related_issues(current_user) + related_issues = source_issue.related_issues(current_user) do |issues| + issues.with_api_entity_associations.preload_awardable + end + related_issues.each { |issue| issue.lazy_subscription(current_user, user_project) } # preload subscriptions present related_issues, with: Entities::RelatedIssue, diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index e348149bb46..f970457fea7 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -52,8 +52,7 @@ module GitalyTest 'RUBYOPT' => nil, # Git hooks can't run during tests as the internal API is not running. - 'GITALY_TESTING_NO_GIT_HOOKS' => "1", - 'GITALY_TESTING_ENABLE_ALL_FEATURE_FLAGS' => "true" + 'GITALY_TESTING_NO_GIT_HOOKS' => "1" } env_hash diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js index d5e6310ed38..28a19fb9df6 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_form_spec.js @@ -20,11 +20,9 @@ describe('Sidebar Confidentiality Form', () => { mutate = jest.fn().mockResolvedValue('Success'), } = {}) => { wrapper = shallowMount(SidebarConfidentialityForm, { - provide: { + propsData: { fullPath: 'group/project', iid: '1', - }, - propsData: { confidential: false, issuableType: 'issue', ...props, diff --git a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js index 20a5be9b518..707215d0739 100644 --- a/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js +++ b/spec/frontend/sidebar/components/confidential/sidebar_confidentiality_widget_spec.js @@ -35,11 +35,11 @@ describe('Sidebar Confidentiality Widget', () => { localVue, apolloProvider: fakeApollo, provide: { - fullPath: 'group/project', - iid: '1', canUpdate: true, }, propsData: { + fullPath: 'group/project', + iid: '1', issuableType: 'issue', }, stubs: { diff --git a/spec/models/concerns/subscribable_spec.rb b/spec/models/concerns/subscribable_spec.rb index e808281b862..a60a0a5e26d 100644 --- a/spec/models/concerns/subscribable_spec.rb +++ b/spec/models/concerns/subscribable_spec.rb @@ -194,10 +194,6 @@ RSpec.describe Subscribable, 'Subscribable' do end end - describe '#lazy_subscribed?' do - it_behaves_like 'returns expected values', :lazy_subscribed? - end - describe '#lazy_subscription' do let(:labels) { create_list(:group_label, 5) } diff --git a/spec/requests/api/issue_links_spec.rb b/spec/requests/api/issue_links_spec.rb index b64d395f25f..45583f5c7dc 100644 --- a/spec/requests/api/issue_links_spec.rb +++ b/spec/requests/api/issue_links_spec.rb @@ -12,26 +12,40 @@ RSpec.describe API::IssueLinks do end describe 'GET /links' do + def perform_request(user = nil, params = {}) + get api("/projects/#{project.id}/issues/#{issue.iid}/links", user), params: params + end + context 'when unauthenticated' do it 'returns 401' do - get api("/projects/#{project.id}/issues/#{issue.iid}/links") + perform_request expect(response).to have_gitlab_http_status(:unauthorized) end end context 'when authenticated' do - it 'returns related issues' do - target_issue = create(:issue, project: project) - create(:issue_link, source: issue, target: target_issue) + let_it_be(:issue_link1) { create(:issue_link, source: issue, target: create(:issue, project: project)) } + let_it_be(:issue_link2) { create(:issue_link, source: issue, target: create(:issue, project: project)) } - get api("/projects/#{project.id}/issues/#{issue.iid}/links", user) + it 'returns related issues' do + perform_request(user) expect(response).to have_gitlab_http_status(:ok) expect(json_response).to be_an Array - expect(json_response.length).to eq(1) + expect(json_response.length).to eq(2) expect(response).to match_response_schema('public_api/v4/issue_links') end + + it 'returns multiple links without N + 1' do + perform_request(user) + + control_count = ActiveRecord::QueryRecorder.new { perform_request(user) }.count + + create(:issue_link, source: issue, target: create(:issue, project: project)) + + expect { perform_request(user) }.not_to exceed_query_limit(control_count) + end end end |