diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-21 15:21:08 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-21 15:21:08 +0300 |
commit | e7198b914bf1d6594909e35d3d00d0a0b260f250 (patch) | |
tree | 6fe636fedeae9c679839d5bf27dc091af037e765 /app | |
parent | 3b80f22aba42e3e424de5c3dd15cc11f96aaac65 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app')
17 files changed, 142 insertions, 205 deletions
diff --git a/app/assets/javascripts/behaviors/components/json_table.vue b/app/assets/javascripts/behaviors/components/json_table.vue index bb38d80c1b5..9cbaa02f270 100644 --- a/app/assets/javascripts/behaviors/components/json_table.vue +++ b/app/assets/javascripts/behaviors/components/json_table.vue @@ -42,6 +42,7 @@ export default { key: field.key, label: field.label, sortable: field.sortable || false, + class: field.class || [], }; }); }, diff --git a/app/assets/javascripts/notebook/cells/output/dataframe_util.js b/app/assets/javascripts/notebook/cells/output/dataframe_util.js index 2fdaaced0b9..41149875d6c 100644 --- a/app/assets/javascripts/notebook/cells/output/dataframe_util.js +++ b/app/assets/javascripts/notebook/cells/output/dataframe_util.js @@ -1,5 +1,70 @@ import { sanitize } from '~/lib/dompurify'; +function parseItems(itemIndexes, itemColumns) { + // Fetching items: if the dataframe has a single column index, the table is simple + // 0: tr > th(index0 value) th(column0 value) th(column1 value) + // 1: tr > th(index0 value) th(column0 value) th(column1 value) + // + // But if the dataframe has multiple column indexes, it uses rowspan, and the row below won't have a value for that + // index. + // 0: tr > th(index0 value, rowspan=2) th(index1 value) th(column0 value) th(column1 value) + // 1: tr > th(index1 value) th(column0 value) th(column1 value) + // + // So, when parsing row 1, and the count of <th> elements is less than indexCount, we fill with the first + // values of row 0 + const indexCount = itemIndexes[0].length; + const rowCount = itemIndexes.length; + + const filledIndexes = itemIndexes.map((row, rowIndex) => { + const indexesInRow = row.length; + if (indexesInRow === indexCount) { + return row; + } + return itemIndexes[rowIndex - 1].slice(0, -indexesInRow).concat(row); + }); + + const items = Array(rowCount); + + for (let row = 0; row < rowCount; row += 1) { + items[row] = { + ...Object.fromEntries(filledIndexes[row].map((value, counter) => [`index${counter}`, value])), + ...Object.fromEntries(itemColumns[row].map((value, counter) => [`column${counter}`, value])), + }; + } + return items; +} + +function labelsToFields(labels, isIndex = true) { + return labels.map((label, counter) => ({ + key: isIndex ? `index${counter}` : `column${counter}`, + label, + sortable: true, + class: isIndex ? 'gl-font-weight-bold' : '', + })); +} + +function parseFields(columnAndIndexLabels, indexCount, columnCount) { + // Fetching the labels: if the dataframe has a single column index, it will be in the format: + // thead + // tr + // th(index0 label) th(column0 label) th(column1 label) + // + // If there are multiple index columns, it the header will actually have two rows: + // thead + // tr + // th() th() th(column 0 label) th(column1 label) + // tr + // th(index0 label) th(index1 label) th() th() + + const columnLabels = columnAndIndexLabels[0].slice(-columnCount); + const indexLabels = columnAndIndexLabels[columnAndIndexLabels.length - 1].slice(0, indexCount); + + const indexFields = labelsToFields(indexLabels, true); + const columnFields = labelsToFields(columnLabels, false); + + return [...indexFields, ...columnFields]; +} + /** * Converts a dataframe in the output of a Jupyter Notebook cell to a json object * @@ -13,27 +78,26 @@ export function convertHtmlTableToJson(input, domParser) { if (!htmlDoc) return { fields: [], items: [] }; - const columnNames = [...htmlDoc.querySelectorAll('table > thead th')].map( - (head) => head.innerText, + const columnAndIndexLabels = [...htmlDoc.querySelectorAll('table > thead tr')].map((row) => + [...row.querySelectorAll('th')].map((item) => item.innerText), ); - if (!columnNames) return { fields: [], items: [] }; + if (columnAndIndexLabels.length === 0) return { fields: [], items: [] }; - const itemValues = [...htmlDoc.querySelectorAll('table > tbody > tr')].map((row) => + const tableRows = [...htmlDoc.querySelectorAll('table > tbody > tr')]; + + const itemColumns = tableRows.map((row) => [...row.querySelectorAll('td')].map((item) => item.innerText), ); - return { - fields: columnNames.map((column) => ({ - key: column === '' ? 'index' : column, - label: column, - sortable: true, - })), - items: itemValues.map((values, itemIndex) => ({ - index: itemIndex, - ...Object.fromEntries(values.map((value, index) => [columnNames[index + 1], value])), - })), - }; + const itemIndexes = tableRows.map((row) => + [...row.querySelectorAll('th')].map((item) => item.innerText), + ); + + const fields = parseFields(columnAndIndexLabels, itemIndexes[0].length, itemColumns[0].length); + const items = parseItems(itemIndexes, itemColumns); + + return { fields, items }; } export function isDataframe(output) { diff --git a/app/assets/javascripts/pages/import/phabricator/new/index.js b/app/assets/javascripts/pages/import/phabricator/new/index.js deleted file mode 100644 index 0bb70a7364e..00000000000 --- a/app/assets/javascripts/pages/import/phabricator/new/index.js +++ /dev/null @@ -1,3 +0,0 @@ -import { initNewProjectUrlSelect } from '~/projects/new'; - -initNewProjectUrlSelect(); diff --git a/app/assets/javascripts/security_configuration/components/constants.js b/app/assets/javascripts/security_configuration/components/constants.js index 1d5ff5eb16f..d46e9983a44 100644 --- a/app/assets/javascripts/security_configuration/components/constants.js +++ b/app/assets/javascripts/security_configuration/components/constants.js @@ -158,6 +158,8 @@ export const LICENSE_COMPLIANCE_HELP_PATH = helpPagePath( 'user/compliance/license_compliance/index', ); +export const CLUSTER_IMAGE_SCANNING_NAME = s__('ciReport|Cluster Image Scanning'); + export const SCANNER_NAMES_MAP = { SAST: SAST_SHORT_NAME, SAST_IAC: SAST_IAC_NAME, @@ -167,7 +169,8 @@ export const SCANNER_NAMES_MAP = { COVERAGE_FUZZING: COVERAGE_FUZZING_NAME, SECRET_DETECTION: SECRET_DETECTION_NAME, DEPENDENCY_SCANNING: DEPENDENCY_SCANNING_NAME, - BAS: BAS_SHORT_NAME, + BREACH_AND_ATTACK_SIMULATION: BAS_NAME, + CLUSTER_IMAGE_SCANNING: CLUSTER_IMAGE_SCANNING_NAME, GENERIC: s__('ciReport|Manually added'), }; diff --git a/app/assets/javascripts/vue_shared/components/file_finder/index.vue b/app/assets/javascripts/vue_shared/components/file_finder/index.vue index 680f229d5e8..5c892377f40 100644 --- a/app/assets/javascripts/vue_shared/components/file_finder/index.vue +++ b/app/assets/javascripts/vue_shared/components/file_finder/index.vue @@ -221,7 +221,12 @@ export default { </script> <template> - <div v-if="visible" class="file-finder-overlay" @mousedown.self="toggle(false)"> + <div + v-if="visible" + data-testid="overlay" + class="file-finder-overlay" + @mousedown.self="toggle(false)" + > <div class="dropdown-menu diff-file-changes file-finder show"> <div :class="{ 'has-value': showClearInputButton }" class="dropdown-input"> <input @@ -231,6 +236,7 @@ export default { type="search" class="dropdown-input-field" autocomplete="off" + data-testid="search-input" @keydown="onKeydown($event)" @keyup="onKeyup($event)" /> @@ -241,6 +247,7 @@ export default { /> <gl-icon name="close" + data-testid="clear-search-input" class="dropdown-input-clear" role="button" :aria-label="__('Clear search input')" diff --git a/app/assets/javascripts/work_items/components/work_item_detail.vue b/app/assets/javascripts/work_items/components/work_item_detail.vue index 361a9ad477b..0e11c04afb1 100644 --- a/app/assets/javascripts/work_items/components/work_item_detail.vue +++ b/app/assets/javascripts/work_items/components/work_item_detail.vue @@ -13,6 +13,7 @@ import { } from '@gitlab/ui'; import noAccessSvg from '@gitlab/svgs/dist/illustrations/analytics/no-access.svg'; import * as Sentry from '@sentry/browser'; +import { fetchPolicies } from '~/lib/graphql'; import { s__ } from '~/locale'; import { parseBoolean } from '~/lib/utils/common_utils'; import { getParameterByName, updateHistory, setUrlParams } from '~/lib/utils/url_utility'; @@ -149,7 +150,13 @@ export default { error() { this.setEmptyState(); }, - result() { + fetchPolicy: fetchPolicies.CACHE_AND_NETWORK, + notifyOnNetworkStatusChange: true, + result(res) { + // need to handle this when the res is loading: true, netWorkStatus: 1, partial: true + if (!res.data) { + return; + } if (isEmpty(this.workItem)) { this.setEmptyState(); } @@ -211,7 +218,7 @@ export default { }, computed: { workItemLoading() { - return this.$apollo.queries.workItem.loading; + return isEmpty(this.workItem) && this.$apollo.queries.workItem.loading; }, workItemType() { return this.workItem.workItemType?.name; @@ -544,7 +551,11 @@ export default { {{ workItemBreadcrumbReference }} </li> </ul> - <div v-else-if="!error" class="gl-mr-auto" data-testid="work-item-type"> + <div + v-else-if="!error && !workItemLoading" + class="gl-mr-auto" + data-testid="work-item-type" + > <work-item-type-icon :work-item-icon-name="workItemIconName" :work-item-type="workItemType && workItemType.toUpperCase()" diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index dbdf4a3055f..a394c59c508 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -64,8 +64,7 @@ class ApplicationController < ActionController::Base :bitbucket_import_enabled?, :bitbucket_import_configured?, :bitbucket_server_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?, - :manifest_import_enabled?, :phabricator_import_enabled?, - :masked_page_url + :manifest_import_enabled?, :masked_page_url def self.endpoint_id_for_action(action_name) "#{name}##{action_name}" @@ -476,10 +475,6 @@ class ApplicationController < ActionController::Base Gitlab::CurrentSettings.import_sources.include?('manifest') end - def phabricator_import_enabled? - Gitlab::PhabricatorImport.available? - end - # U2F (universal 2nd factor) devices need a unique identifier for the application # to perform authentication. # https://developers.yubico.com/U2F/App_ID.html diff --git a/app/controllers/import/phabricator_controller.rb b/app/controllers/import/phabricator_controller.rb deleted file mode 100644 index d1c04817689..00000000000 --- a/app/controllers/import/phabricator_controller.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -class Import::PhabricatorController < Import::BaseController - include ImportHelper - - before_action :verify_import_enabled - - def new - end - - def create - @project = Gitlab::PhabricatorImport::ProjectCreator - .new(current_user, import_params).execute - - if @project&.persisted? - redirect_to @project - else - @name = params[:name] - @path = params[:path] - @errors = @project&.errors&.full_messages || [_("Invalid import params")] - - render :new - end - end - - def verify_import_enabled - render_404 unless phabricator_import_enabled? - end - - private - - def import_params - params.permit(:path, :phabricator_server_url, :api_token, :name, :namespace_id) - end -end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 52abacfe3e8..196000ad441 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -738,6 +738,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord before_validation :ensure_uuid! before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? before_validation :normalize_default_branch_name + before_validation :remove_old_import_sources before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -781,6 +782,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord users_count >= INSTANCE_REVIEW_MIN_USERS end + def remove_old_import_sources + self.import_sources -= ['phabricator'] if self.import_sources + end + Recursion = Class.new(RuntimeError) def self.create_from_defaults diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 77974a0f36b..6a932939ae9 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -45,7 +45,8 @@ class Namespace::RootStorageStatistics < ApplicationRecord attributes_from_project_statistics.merge!( attributes_from_personal_snippets, attributes_from_namespace_statistics, - attributes_for_container_registry_size + attributes_for_container_registry_size, + attributes_for_forks_statistics ) { |key, v1, v2| v1 + v2 } end @@ -58,6 +59,32 @@ class Namespace::RootStorageStatistics < ApplicationRecord }.with_indifferent_access end + def attributes_for_forks_statistics + return {} unless ::Feature.enabled?(:root_storage_statistics_calculate_forks, namespace) + + visibility_levels_to_storage_size_columns = { + Gitlab::VisibilityLevel::PRIVATE => :private_forks_storage_size, + Gitlab::VisibilityLevel::INTERNAL => :internal_forks_storage_size, + Gitlab::VisibilityLevel::PUBLIC => :public_forks_storage_size + } + + defaults = { + private_forks_storage_size: 0, + internal_forks_storage_size: 0, + public_forks_storage_size: 0 + } + + defaults.merge(for_forks_statistics.transform_keys { |k| visibility_levels_to_storage_size_columns[k] }) + end + + def for_forks_statistics + all_projects + .joins([:statistics, :fork_network]) + .where('fork_networks.root_project_id != projects.id') + .group('projects.visibility_level') + .sum('project_statistics.storage_size') + end + def attributes_from_project_statistics from_project_statistics .take diff --git a/app/views/import/phabricator/new.html.haml b/app/views/import/phabricator/new.html.haml deleted file mode 100644 index d4fdd107043..00000000000 --- a/app/views/import/phabricator/new.html.haml +++ /dev/null @@ -1,26 +0,0 @@ -- page_title _('Phabricator Server Import') -- header_title _("New project"), new_project_path -- add_to_breadcrumbs s_('ProjectsNew|Import project'), new_project_path(anchor: 'import_project') - -%h1.page-title.gl-font-size-h-display.d-flex - .gl-display-flex.gl-align-items-center.gl-justify-content-center - = sprite_icon('issues', css_class: 'gl-mr-2') - = _('Import tasks from Phabricator into issues') - -= render 'import/shared/errors' - -= form_tag import_phabricator_path, class: 'new_project', method: :post do - = render 'import/shared/new_project_form' - - %h4.gl-mt-0= _('Enter in your Phabricator Server URL and personal access token below') - - .form-group.row - = label_tag :phabricator_server_url, _('Phabricator Server URL'), class: 'col-form-label col-md-2' - .col-md-4 - = text_field_tag :phabricator_server_url, params[:phabricator_server_url], class: 'form-control gl-form-input gl-mr-3', placeholder: 'https://your-phabricator-server', size: 40 - .form-group.row - = label_tag :api_token, _('API Token'), class: 'col-form-label col-md-2' - .col-md-4 - = password_field_tag :api_token, params[:api_token], class: 'form-control gl-form-input gl-mr-3', placeholder: _('Personal Access Token'), size: 40 - .form-actions - = submit_tag _('Import tasks'), class: 'gl-button btn btn-confirm' diff --git a/app/views/notify/pages_domain_auto_ssl_failed_email.html.haml b/app/views/notify/pages_domain_auto_ssl_failed_email.html.haml index 1bc2cc15616..c0b334fba94 100644 --- a/app/views/notify/pages_domain_auto_ssl_failed_email.html.haml +++ b/app/views/notify/pages_domain_auto_ssl_failed_email.html.haml @@ -8,4 +8,4 @@ - docs_url = help_page_url('user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md', anchor: 'troubleshooting') - link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: docs_url } - link_end = '</a>'.html_safe - = _("Please follow the %{link_start}Let\'s Encrypt troubleshooting instructions%{link_end} to re-obtain your Let's Encrypt certificate.").html_safe % { link_start: link_start, link_end: link_end } + = _("Please follow the %{link_start}Let's Encrypt troubleshooting instructions%{link_end} to re-obtain your Let's Encrypt certificate.").html_safe % { link_start: link_start, link_end: link_end } diff --git a/app/views/notify/pages_domain_auto_ssl_failed_email.text.haml b/app/views/notify/pages_domain_auto_ssl_failed_email.text.haml index 6f20d11c966..feb88d2df39 100644 --- a/app/views/notify/pages_domain_auto_ssl_failed_email.text.haml +++ b/app/views/notify/pages_domain_auto_ssl_failed_email.text.haml @@ -4,4 +4,4 @@ #{_('Domain')}: #{project_pages_domain_url(@project, @domain)} - docs_url = help_page_url('user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md', anchor: 'troubleshooting') -= _("Please follow the Let\'s Encrypt troubleshooting instructions to re-obtain your Let's Encrypt certificate: %{docs_url}.").html_safe % { docs_url: docs_url } += _("Please follow the Let's Encrypt troubleshooting instructions to re-obtain your Let's Encrypt certificate: %{docs_url}.").html_safe % { docs_url: docs_url } diff --git a/app/views/projects/_import_project_pane.html.haml b/app/views/projects/_import_project_pane.html.haml index 412c91544a6..f40b31d09f1 100644 --- a/app/views/projects/_import_project_pane.html.haml +++ b/app/views/projects/_import_project_pane.html.haml @@ -63,11 +63,6 @@ = render Pajamas::ButtonComponent.new(href: new_import_manifest_path(namespace_id: namespace_id), icon: 'doc-text', button_options: { class: 'import_manifest js-import-project-btn', data: { platform: 'manifest_file', **tracking_attrs_data(track_label, 'click_button', 'manifest_file') } }) do = _('Manifest file') - - if phabricator_import_enabled? - %div - = render Pajamas::ButtonComponent.new(href: new_import_phabricator_path(namespace_id: namespace_id), icon: 'issues', button_options: { class: 'import_phabricator js-import-project-btn', data: { platform: 'phabricator', track_label: "#{track_label}", track_action: "click_button", track_property: "phabricator" } }) do - = _('Phabricator tasks') - = render_if_exists "projects/gitee_import_button", namespace_id: namespace_id, track_label: track_label diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f47d5da95f0..582f5037e48 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -3081,15 +3081,6 @@ :weight: 1 :idempotent: true :tags: [] -- :name: phabricator_import_import_tasks - :worker_name: Gitlab::PhabricatorImport::ImportTasksWorker - :feature_category: :importers - :has_external_dependencies: false - :urgency: :low - :resource_boundary: :unknown - :weight: 1 - :idempotent: false - :tags: [] - :name: post_receive :worker_name: PostReceive :feature_category: :source_code_management diff --git a/app/workers/gitlab/phabricator_import/base_worker.rb b/app/workers/gitlab/phabricator_import/base_worker.rb deleted file mode 100644 index 2dc4855f854..00000000000 --- a/app/workers/gitlab/phabricator_import/base_worker.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -# All workers within a Phabricator import should inherit from this worker and -# implement the `#import` method. The jobs should then be scheduled using the -# `.schedule` class method instead of `.perform_async` -# -# Doing this makes sure that only one job of that type is running at the same time -# for a certain project. This will avoid deadlocks. When a job is already running -# we'll wait for it for 10 times 5 seconds to restart. If the running job hasn't -# finished, by then, we'll retry in 30 seconds. -# -# It also makes sure that we keep the import state of the project up to date: -# - It keeps track of the jobs so we know how many jobs are running for the -# project -# - It refreshes the import jid, so it doesn't get cleaned up by the -# `Gitlab::Import::StuckProjectImportJobsWorker` -# - It marks the import as failed if a job failed to many times -# - It marks the import as finished when all remaining jobs are done -module Gitlab - module PhabricatorImport - class BaseWorker # rubocop:disable Scalability/IdempotentWorker - include WorkerAttributes - include Gitlab::ExclusiveLeaseHelpers - - feature_category :importers - - class << self - def schedule(project_id, *args) - perform_async(project_id, *args) - add_job(project_id) - end - - def add_job(project_id) - worker_state(project_id).add_job - end - - def remove_job(project_id) - worker_state(project_id).remove_job - end - - def worker_state(project_id) - Gitlab::PhabricatorImport::WorkerState.new(project_id) - end - end - - def perform(project_id, *args) - in_lock("#{self.class.name.underscore}/#{project_id}/#{args}", ttl: 2.hours, sleep_sec: 5.seconds) do - project = Project.find_by_id(project_id) - next unless project - - # Bail if the import job already failed - next unless project.import_state&.in_progress? - - project.import_state.refresh_jid_expiration - - import(project, *args) - - # If this is the last running job, finish the import - project.after_import if self.class.worker_state(project_id).running_count < 2 - - self.class.remove_job(project_id) - end - rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError - # Reschedule a job if there was already a running one - # Running them at the same time could cause a deadlock updating the same - # resource - self.class.perform_in(30.seconds, project_id, *args) - end - - private - - def import(project, *args) - importer_class.new(project, *args).execute - end - - def importer_class - raise NotImplementedError, "Implement `#{__method__}` on #{self.class}" - end - end - end -end diff --git a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb b/app/workers/gitlab/phabricator_import/import_tasks_worker.rb deleted file mode 100644 index f650681fc2f..00000000000 --- a/app/workers/gitlab/phabricator_import/import_tasks_worker.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - class ImportTasksWorker < BaseWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - data_consistency :always - - sidekiq_options retry: 3 - include ProjectImportOptions # This marks the project as failed after too many tries - - def importer_class - Gitlab::PhabricatorImport::Issues::Importer - end - end - end -end |