diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-29 12:09:27 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-29 12:09:27 +0300 |
commit | 518aebfe14a72ba08b5b0a528654805bb2d66cd3 (patch) | |
tree | 9e0147148b3c512d69ef941de6d25ae41fcbcfeb | |
parent | 7ee1ce8d781f506180501c5eb07d1dfcfb8b2c6a (diff) |
Add latest changes from gitlab-org/gitlab@master
37 files changed, 459 insertions, 401 deletions
diff --git a/app/assets/javascripts/groups/members/index.js b/app/assets/javascripts/groups/members/index.js index 68caf6628f6..9ce0e3c1179 100644 --- a/app/assets/javascripts/groups/members/index.js +++ b/app/assets/javascripts/groups/members/index.js @@ -5,7 +5,16 @@ import { parseDataAttributes } from 'ee_else_ce/groups/members/utils'; import App from './components/app.vue'; import membersStore from '~/members/store'; -export const initGroupMembersApp = (el, tableFields, tableAttrs, requestFormatter) => { +export const initGroupMembersApp = ( + el, + { + tableFields = [], + tableAttrs = {}, + tableSortableFields = [], + requestFormatter = () => {}, + filteredSearchBar = { show: false }, + }, +) => { if (!el) { return () => {}; } @@ -19,7 +28,9 @@ export const initGroupMembersApp = (el, tableFields, tableAttrs, requestFormatte currentUserId: gon.current_user_id || null, tableFields, tableAttrs, + tableSortableFields, requestFormatter, + filteredSearchBar, }), ); diff --git a/app/assets/javascripts/groups/members/utils.js b/app/assets/javascripts/groups/members/utils.js index 662eecc4e38..2d584556bbc 100644 --- a/app/assets/javascripts/groups/members/utils.js +++ b/app/assets/javascripts/groups/members/utils.js @@ -1,5 +1,5 @@ import { isUndefined } from 'lodash'; -import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; +import { convertObjectPropsToCamelCase, parseBoolean } from '~/lib/utils/common_utils'; import { GROUP_MEMBER_BASE_PROPERTY_NAME, GROUP_MEMBER_ACCESS_LEVEL_PROPERTY_NAME, @@ -8,12 +8,13 @@ import { } from './constants'; export const parseDataAttributes = el => { - const { members, groupId, memberPath } = el.dataset; + const { members, groupId, memberPath, canManageMembers } = el.dataset; return { members: convertObjectPropsToCamelCase(JSON.parse(members), { deep: true }), sourceId: parseInt(groupId, 10), memberPath, + canManageMembers: parseBoolean(canManageMembers), }; }; diff --git a/app/assets/javascripts/members/store/state.js b/app/assets/javascripts/members/store/state.js index ab3ebb34616..23a7983adcc 100644 --- a/app/assets/javascripts/members/store/state.js +++ b/app/assets/javascripts/members/store/state.js @@ -2,18 +2,24 @@ export default ({ members, sourceId, currentUserId, + canManageMembers, tableFields, tableAttrs, + tableSortableFields, memberPath, requestFormatter, + filteredSearchBar, }) => ({ members, sourceId, currentUserId, + canManageMembers, tableFields, tableAttrs, + tableSortableFields, memberPath, requestFormatter, + filteredSearchBar, showError: false, errorMessage: '', removeGroupLinkModalVisible: false, diff --git a/app/assets/javascripts/pages/groups/group_members/index.js b/app/assets/javascripts/pages/groups/group_members/index.js index 009a3eee526..fbb960a7ceb 100644 --- a/app/assets/javascripts/pages/groups/group_members/index.js +++ b/app/assets/javascripts/pages/groups/group_members/index.js @@ -6,6 +6,7 @@ import groupsSelect from '~/groups_select'; import RemoveMemberModal from '~/vue_shared/components/remove_member_modal.vue'; import { initGroupMembersApp } from '~/groups/members'; import { memberRequestFormatter, groupLinkRequestFormatter } from '~/groups/members/utils'; +import { __ } from '~/locale'; function mountRemoveMemberModal() { const el = document.querySelector('.js-remove-member-modal'); @@ -22,30 +23,43 @@ function mountRemoveMemberModal() { } const SHARED_FIELDS = ['account', 'expires', 'maxRole', 'expiration', 'actions']; -initGroupMembersApp( - document.querySelector('.js-group-members-list'), - SHARED_FIELDS.concat(['source', 'granted']), - { tr: { 'data-qa-selector': 'member_row' } }, - memberRequestFormatter, -); -initGroupMembersApp( - document.querySelector('.js-group-linked-list'), - SHARED_FIELDS.concat('granted'), - { table: { 'data-qa-selector': 'groups_list' }, tr: { 'data-qa-selector': 'group_row' } }, - groupLinkRequestFormatter, -); -initGroupMembersApp( - document.querySelector('.js-group-invited-members-list'), - SHARED_FIELDS.concat('invited'), - {}, - memberRequestFormatter, -); -initGroupMembersApp( - document.querySelector('.js-group-access-requests-list'), - SHARED_FIELDS.concat('requested'), - {}, - memberRequestFormatter, -); + +initGroupMembersApp(document.querySelector('.js-group-members-list'), { + tableFields: SHARED_FIELDS.concat(['source', 'granted']), + tableAttrs: { tr: { 'data-qa-selector': 'member_row' } }, + tableSortableFields: ['account', 'granted', 'maxRole', 'lastSignIn'], + requestFormatter: memberRequestFormatter, + filteredSearchBar: { + show: true, + tokens: ['two_factor', 'with_inherited_permissions'], + searchParam: 'search', + placeholder: __('Members|Filter members'), + recentSearchesStorageKey: 'group_members', + }, +}); +initGroupMembersApp(document.querySelector('.js-group-linked-list'), { + tableFields: SHARED_FIELDS.concat('granted'), + tableAttrs: { + table: { 'data-qa-selector': 'groups_list' }, + tr: { 'data-qa-selector': 'group_row' }, + }, + requestFormatter: groupLinkRequestFormatter, +}); +initGroupMembersApp(document.querySelector('.js-group-invited-members-list'), { + tableFields: SHARED_FIELDS.concat('invited'), + requestFormatter: memberRequestFormatter, + filteredSearchBar: { + show: true, + tokens: [], + searchParam: 'search_invited', + placeholder: __('Members|Search invited'), + recentSearchesStorageKey: 'group_invited_members', + }, +}); +initGroupMembersApp(document.querySelector('.js-group-access-requests-list'), { + tableFields: SHARED_FIELDS.concat('requested'), + requestFormatter: memberRequestFormatter, +}); groupsSelect(); memberExpirationDate(); diff --git a/app/assets/javascripts/pages/projects/blob/show/index.js b/app/assets/javascripts/pages/projects/blob/show/index.js index 74c1c2e981e..a96b88732b4 100644 --- a/app/assets/javascripts/pages/projects/blob/show/index.js +++ b/app/assets/javascripts/pages/projects/blob/show/index.js @@ -6,30 +6,6 @@ import GpgBadges from '~/gpg_badges'; import initWebIdeLink from '~/pages/projects/shared/web_ide_link'; import '~/sourcegraph/load'; import PipelineTourSuccessModal from '~/blob/pipeline_tour_success_modal.vue'; -import { parseBoolean } from '~/lib/utils/common_utils'; - -const createGitlabCiYmlVisualization = (containerId = '#js-blob-toggle-graph-preview') => { - const el = document.querySelector(containerId); - const { isCiConfigFile, blobData } = el?.dataset; - - if (el && parseBoolean(isCiConfigFile)) { - // eslint-disable-next-line no-new - new Vue({ - el, - components: { - GitlabCiYamlVisualization: () => - import('~/pipelines/components/pipeline_graph/gitlab_ci_yaml_visualization.vue'), - }, - render(createElement) { - return createElement('gitlabCiYamlVisualization', { - props: { - blobData, - }, - }); - }, - }); - } -}; document.addEventListener('DOMContentLoaded', () => { new BlobViewer(); // eslint-disable-line no-new @@ -88,8 +64,4 @@ document.addEventListener('DOMContentLoaded', () => { }, }); } - - if (gon?.features?.gitlabCiYmlPreview) { - createGitlabCiYmlVisualization(); - } }); diff --git a/app/assets/javascripts/pipelines/components/pipeline_graph/gitlab_ci_yaml_visualization.vue b/app/assets/javascripts/pipelines/components/pipeline_graph/gitlab_ci_yaml_visualization.vue deleted file mode 100644 index 3cc76425e2a..00000000000 --- a/app/assets/javascripts/pipelines/components/pipeline_graph/gitlab_ci_yaml_visualization.vue +++ /dev/null @@ -1,76 +0,0 @@ -<script> -import { GlTab, GlTabs } from '@gitlab/ui'; -import jsYaml from 'js-yaml'; -import PipelineGraph from './pipeline_graph.vue'; -import { preparePipelineGraphData } from '../../utils'; - -export default { - FILE_CONTENT_SELECTOR: '#blob-content', - EMPTY_FILE_SELECTOR: '.nothing-here-block', - - components: { - GlTab, - GlTabs, - PipelineGraph, - }, - props: { - blobData: { - required: true, - type: String, - }, - }, - data() { - return { - selectedTabIndex: 0, - pipelineData: {}, - }; - }, - computed: { - isVisualizationTab() { - return this.selectedTabIndex === 1; - }, - }, - async created() { - if (this.blobData) { - // The blobData in this case represents the gitlab-ci.yml data - const json = await jsYaml.load(this.blobData); - this.pipelineData = preparePipelineGraphData(json); - } - }, - methods: { - // This is used because the blob page still uses haml, and we can't make - // our haml hide the unused section so we resort to a standard query here. - toggleFileContent({ isFileTab }) { - const el = document.querySelector(this.$options.FILE_CONTENT_SELECTOR); - const emptySection = document.querySelector(this.$options.EMPTY_FILE_SELECTOR); - - const elementToHide = el || emptySection; - - if (!elementToHide) { - return; - } - - // Checking for the current style display prevents user - // from toggling visiblity on and off when clicking on the tab - if (!isFileTab && elementToHide.style.display !== 'none') { - elementToHide.style.display = 'none'; - } - - if (isFileTab && elementToHide.style.display === 'none') { - elementToHide.style.display = 'block'; - } - }, - }, -}; -</script> -<template> - <div> - <div> - <gl-tabs v-model="selectedTabIndex"> - <gl-tab :title="__('File')" @click="toggleFileContent({ isFileTab: true })" /> - <gl-tab :title="__('Visualization')" @click="toggleFileContent({ isFileTab: false })" /> - </gl-tabs> - </div> - <pipeline-graph v-if="isVisualizationTab" :pipeline-data="pipelineData" /> - </div> -</template> diff --git a/app/assets/javascripts/pipelines/utils.js b/app/assets/javascripts/pipelines/utils.js index 7d1a1762e0d..46e54bfb4ff 100644 --- a/app/assets/javascripts/pipelines/utils.js +++ b/app/assets/javascripts/pipelines/utils.js @@ -7,63 +7,6 @@ export const validateParams = params => { export const createUniqueJobId = (stageName, jobName) => `${stageName}-${jobName}`; -/** - * This function takes a json payload that comes from a yml - * file converted to json through `jsyaml` library. Because we - * naively convert the entire yaml to json, some keys (like `includes`) - * are irrelevant to rendering the graph and must be removed. We also - * restructure the data to have the structure from an API response for the - * pipeline data. - * @param {Object} jsonData - * @returns {Array} - Array of stages containing all jobs - */ -export const preparePipelineGraphData = jsonData => { - const jsonKeys = Object.keys(jsonData); - const jobNames = jsonKeys.filter(job => jsonData[job]?.stage); - // Creates an object with only the valid jobs - const jobs = jsonKeys.reduce((acc, val) => { - if (jobNames.includes(val)) { - return { - ...acc, - [val]: { ...jsonData[val], id: createUniqueJobId(jsonData[val].stage, val) }, - }; - } - return { ...acc }; - }, {}); - - // We merge both the stages from the "stages" key in the yaml and the stage associated - // with each job to show the user both the stages they explicitly defined, and those - // that they added under jobs. We also remove duplicates. - const jobStages = jobNames.map(job => jsonData[job].stage); - const userDefinedStages = jsonData?.stages ?? []; - - // The order is important here. We always show the stages in order they were - // defined in the `stages` key first, and then stages that are under the jobs. - const stages = Array.from(new Set([...userDefinedStages, ...jobStages])); - - const arrayOfJobsByStage = stages.map(val => { - return jobNames.filter(job => { - return jsonData[job].stage === val; - }); - }); - - const pipelineData = stages.map((stage, index) => { - const stageJobs = arrayOfJobsByStage[index]; - return { - name: stage, - groups: stageJobs.map(job => { - return { - name: job, - jobs: [{ ...jsonData[job] }], - id: createUniqueJobId(stage, job), - }; - }), - }; - }); - - return { stages: pipelineData, jobs }; -}; - export const generateJobNeedsDict = ({ jobs }) => { const arrOfJobNames = Object.keys(jobs); diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index d1c0f7edc5c..8f16650a6f2 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -32,10 +32,6 @@ class Projects::BlobController < Projects::ApplicationController before_action :validate_diff_params, only: :diff before_action :set_last_commit_sha, only: [:edit, :update] - before_action only: :show do - push_frontend_feature_flag(:gitlab_ci_yml_preview, @project, default_enabled: false) - end - track_redis_hll_event :create, :update, name: 'g_edit_by_sfe', feature: :track_editor_edit_actions, feature_default_enabled: true feature_category :source_code_management diff --git a/app/helpers/groups/group_members_helper.rb b/app/helpers/groups/group_members_helper.rb index ee90585112b..adc9d85a384 100644 --- a/app/helpers/groups/group_members_helper.rb +++ b/app/helpers/groups/group_members_helper.rb @@ -26,7 +26,8 @@ module Groups::GroupMembersHelper { members: members_data_json(group, members), member_path: group_group_member_path(group, ':id'), - group_id: group.id + group_id: group.id, + can_manage_members: can?(current_user, :admin_group_member, group).to_s } end diff --git a/app/views/projects/blob/_content.html.haml b/app/views/projects/blob/_content.html.haml index 5b77e31eb00..7afbd85cd6d 100644 --- a/app/views/projects/blob/_content.html.haml +++ b/app/views/projects/blob/_content.html.haml @@ -1,10 +1,6 @@ - simple_viewer = blob.simple_viewer - rich_viewer = blob.rich_viewer - rich_viewer_active = rich_viewer && params[:viewer] != 'simple' -- blob_data = defined?(@blob) ? @blob.data : {} -- is_ci_config_file = defined?(@blob) && defined?(@project) ? editing_ci_config?.to_s : 'false' - -#js-blob-toggle-graph-preview{ data: { blob_data: blob_data, is_ci_config_file: is_ci_config_file } } = render 'projects/blob/viewer', viewer: simple_viewer, hidden: rich_viewer_active diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index 63c1ba8e699..575cd4862b0 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -15,17 +15,25 @@ module Gitlab feature_category :importers worker_has_external_dependencies! + + def logger + @logger ||= Gitlab::Import::Logger.build + end end # project - An instance of `Project` to import the data into. # client - An instance of `Gitlab::GithubImport::Client` # hash - A Hash containing the details of the object to import. def import(project, client, hash) - object = representation_class.from_json_hash(hash) + info(project.id, message: 'starting importer') + object = representation_class.from_json_hash(hash) importer_class.new(object, project, client).execute counter.increment + info(project.id, message: 'importer finished') + rescue => e + error(project.id, e) end def counter @@ -52,6 +60,35 @@ module Gitlab def counter_description raise NotImplementedError end + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + importer: importer_class.name + ) + end end end end diff --git a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb index 1c6413674a0..eb1af0869bd 100644 --- a/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb +++ b/app/workers/concerns/gitlab/github_import/rescheduling_methods.rb @@ -6,7 +6,7 @@ module Gitlab # importing GitHub projects. module ReschedulingMethods # project_id - The ID of the GitLab project to import the note into. - # hash - A Hash containing the details of the GitHub object to imoprt. + # hash - A Hash containing the details of the GitHub object to import. # notify_key - The Redis key to notify upon completion, if any. # rubocop: disable CodeReuse/ActiveRecord def perform(project_id, hash, notify_key = nil) diff --git a/app/workers/concerns/gitlab/github_import/stage_methods.rb b/app/workers/concerns/gitlab/github_import/stage_methods.rb index e2dee315cde..e5985fb94da 100644 --- a/app/workers/concerns/gitlab/github_import/stage_methods.rb +++ b/app/workers/concerns/gitlab/github_import/stage_methods.rb @@ -5,11 +5,17 @@ module Gitlab module StageMethods # project_id - The ID of the GitLab project to import the data into. def perform(project_id) + info(project_id, message: 'starting stage') + return unless (project = find_project(project_id)) client = GithubImport.new_client_for(project) try_import(client, project) + + info(project_id, message: 'stage finished') + rescue => e + error(project_id, e) end # client - An instance of Gitlab::GithubImport::Client. @@ -27,6 +33,39 @@ module Gitlab Project.joins_import_state.where(import_state: { status: :started }).find_by(id: id) end # rubocop: enable CodeReuse/ActiveRecord + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'stage failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + import_stage: self.class.name + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end diff --git a/app/workers/gitlab/github_import/stage/finish_import_worker.rb b/app/workers/gitlab/github_import/stage/finish_import_worker.rb index 73699a74a4a..058e1a0853d 100644 --- a/app/workers/gitlab/github_import/stage/finish_import_worker.rb +++ b/app/workers/gitlab/github_import/stage/finish_import_worker.rb @@ -20,12 +20,15 @@ module Gitlab def report_import_time(project) duration = Time.zone.now - project.created_at - path = project.full_path - histogram.observe({ project: path }, duration) + histogram.observe({ project: project.full_path }, duration) counter.increment - logger.info("GitHub importer finished for #{path} in #{duration.round(2)} seconds") + info( + project.id, + message: "GitHub project import finished", + duration_s: duration.round(2) + ) end def histogram diff --git a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb index 11c2a2ac9b4..202bb335ca1 100644 --- a/app/workers/gitlab/github_import/stage/import_base_data_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_base_data_worker.rb @@ -20,6 +20,7 @@ module Gitlab # project - An instance of Project. def import(client, project) IMPORTERS.each do |klass| + info(project.id, message: "starting importer", importer: klass.name) klass.new(project, client).execute end diff --git a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb index 68b6e159fa4..486057804b4 100644 --- a/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_issues_and_diff_notes_worker.rb @@ -19,6 +19,7 @@ module Gitlab # project - An instance of Project. def import(client, project) waiters = IMPORTERS.each_with_object({}) do |klass, hash| + info(project.id, message: "starting importer", importer: klass.name) waiter = klass.new(project, client).execute hash[waiter.key] = waiter.jobs_remaining end diff --git a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb index a19df399969..de2a7f9fc29 100644 --- a/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_lfs_objects_worker.rb @@ -16,6 +16,8 @@ module Gitlab # project - An instance of Project. def import(project) + info(project.id, message: "starting importer", importer: 'Importer::LfsObjectsImporter') + waiter = Importer::LfsObjectsImporter .new(project, nil) .execute diff --git a/app/workers/gitlab/github_import/stage/import_notes_worker.rb b/app/workers/gitlab/github_import/stage/import_notes_worker.rb index 49b9821cd45..e1da26a9d48 100644 --- a/app/workers/gitlab/github_import/stage/import_notes_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_notes_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::NotesImporter') waiter = Importer::NotesImporter .new(project, client) .execute diff --git a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb index d32111f4aa6..bf2defa6326 100644 --- a/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_pull_requests_worker.rb @@ -11,6 +11,7 @@ module Gitlab # client - An instance of Gitlab::GithubImport::Client. # project - An instance of Project. def import(client, project) + info(project.id, message: "starting importer", importer: 'Importer::PullRequestsImporter') waiter = Importer::PullRequestsImporter .new(project, client) .execute diff --git a/app/workers/gitlab/github_import/stage/import_repository_worker.rb b/app/workers/gitlab/github_import/stage/import_repository_worker.rb index cb9ef1cd198..3338f7e58c0 100644 --- a/app/workers/gitlab/github_import/stage/import_repository_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_repository_worker.rb @@ -21,6 +21,7 @@ module Gitlab # expiration time. RefreshImportJidWorker.perform_in_the_future(project.id, jid) + info(project.id, message: "starting importer", importer: 'Importer::RepositoryImporter') importer = Importer::RepositoryImporter.new(project, client) return unless importer.execute diff --git a/config/feature_flags/development/gitlab_ci_yml_preview.yml b/config/feature_flags/development/gitlab_ci_yml_preview.yml deleted file mode 100644 index 5b5453c0f08..00000000000 --- a/config/feature_flags/development/gitlab_ci_yml_preview.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: gitlab_ci_yml_preview -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40880 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/244905 -milestone: '13.4' -type: development -group: group::ci -default_enabled: false diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 5980b3c2179..0024a2e7dc6 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -28,11 +28,6 @@ module Gitlab lfs_objects.each do |object| yield object end - rescue StandardError => e - Gitlab::Import::Logger.error( - message: 'The Lfs import process failed', - error: e.message - ) end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cabc615ea11..d37b4153524 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -26,6 +26,8 @@ module Gitlab end def execute + info(project.id, message: "starting importer") + retval = if parallel? parallel_import @@ -43,8 +45,11 @@ module Gitlab # completed those jobs will just cycle through any remaining pages while # not scheduling anything. Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) + info(project.id, message: "importer finished") retval + rescue => e + error(project.id, e) end # Imports all the objects in sequence in the current thread. @@ -157,6 +162,40 @@ module Gitlab def collection_options {} end + + private + + def info(project_id, extra = {}) + logger.info(log_attributes(project_id, extra)) + end + + def error(project_id, exception) + logger.error( + log_attributes( + project_id, + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_and_raise_exception( + exception, + log_attributes(project_id) + ) + end + + def log_attributes(project_id, extra = {}) + extra.merge( + import_source: :github, + project_id: project_id, + importer: importer_class.name, + parallel: parallel? + ) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 29886cd118a..fd5c0d99eb3 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -16848,6 +16848,9 @@ msgstr "" msgid "Members|Expired" msgstr "" +msgid "Members|Filter members" +msgstr "" + msgid "Members|LDAP override enabled." msgstr "" @@ -16872,6 +16875,9 @@ msgstr "" msgid "Members|Role updated successfully." msgstr "" +msgid "Members|Search invited" +msgstr "" + msgid "Members|in %{time}" msgstr "" @@ -30145,9 +30151,6 @@ msgstr "" msgid "VisualReviewApp|Steps 1 and 2 (and sometimes 3) are performed once by the developer before requesting feedback. Steps 3 (if necessary), 4 is performed by the reviewer each time they perform a review." msgstr "" -msgid "Visualization" -msgstr "" - msgid "Vulnerabilities" msgstr "" diff --git a/spec/frontend/groups/members/index_spec.js b/spec/frontend/groups/members/index_spec.js index aaa36665c45..5c717e53229 100644 --- a/spec/frontend/groups/members/index_spec.js +++ b/spec/frontend/groups/members/index_spec.js @@ -9,12 +9,13 @@ describe('initGroupMembersApp', () => { let wrapper; const setup = () => { - vm = initGroupMembersApp( - el, - ['account'], - { table: { 'data-qa-selector': 'members_list' } }, - () => ({}), - ); + vm = initGroupMembersApp(el, { + tableFields: ['account'], + tableAttrs: { table: { 'data-qa-selector': 'members_list' } }, + tableSortableFields: ['account'], + requestFormatter: () => ({}), + filteredSearchBar: { show: false }, + }); wrapper = createWrapper(vm); }; @@ -22,6 +23,7 @@ describe('initGroupMembersApp', () => { el = document.createElement('div'); el.setAttribute('data-members', membersJsonString); el.setAttribute('data-group-id', '234'); + el.setAttribute('data-can-manage-members', 'true'); el.setAttribute('data-member-path', '/groups/foo-bar/-/group_members/:id'); window.gon = { current_user_id: 123 }; @@ -61,6 +63,12 @@ describe('initGroupMembersApp', () => { expect(vm.$store.state.sourceId).toBe(234); }); + it('parses and sets `data-can-manage-members` as `canManageMembers` in Vuex store', () => { + setup(); + + expect(vm.$store.state.canManageMembers).toBe(true); + }); + it('parses and sets `members` in Vuex store', () => { setup(); @@ -79,12 +87,24 @@ describe('initGroupMembersApp', () => { expect(vm.$store.state.tableAttrs).toEqual({ table: { 'data-qa-selector': 'members_list' } }); }); + it('sets `tableSortableFields` in Vuex store', () => { + setup(); + + expect(vm.$store.state.tableSortableFields).toEqual(['account']); + }); + it('sets `requestFormatter` in Vuex store', () => { setup(); expect(vm.$store.state.requestFormatter()).toEqual({}); }); + it('sets `filteredSearchBar` in Vuex store', () => { + setup(); + + expect(vm.$store.state.filteredSearchBar).toEqual({ show: false }); + }); + it('sets `memberPath` in Vuex store', () => { setup(); diff --git a/spec/frontend/groups/members/utils_spec.js b/spec/frontend/groups/members/utils_spec.js index b0921c7642f..68945174e9d 100644 --- a/spec/frontend/groups/members/utils_spec.js +++ b/spec/frontend/groups/members/utils_spec.js @@ -13,6 +13,7 @@ describe('group member utils', () => { el = document.createElement('div'); el.setAttribute('data-members', membersJsonString); el.setAttribute('data-group-id', '234'); + el.setAttribute('data-can-manage-members', 'true'); }); afterEach(() => { @@ -23,6 +24,7 @@ describe('group member utils', () => { expect(parseDataAttributes(el)).toEqual({ members: membersParsed, sourceId: 234, + canManageMembers: true, }); }); }); diff --git a/spec/frontend/pipelines/pipeline_graph/gitlab_ci_yaml_visualization_spec.js b/spec/frontend/pipelines/pipeline_graph/gitlab_ci_yaml_visualization_spec.js deleted file mode 100644 index fea42350959..00000000000 --- a/spec/frontend/pipelines/pipeline_graph/gitlab_ci_yaml_visualization_spec.js +++ /dev/null @@ -1,47 +0,0 @@ -import { shallowMount } from '@vue/test-utils'; -import { GlTab } from '@gitlab/ui'; -import { yamlString } from './mock_data'; -import PipelineGraph from '~/pipelines/components/pipeline_graph/pipeline_graph.vue'; -import GitlabCiYamlVisualization from '~/pipelines/components/pipeline_graph/gitlab_ci_yaml_visualization.vue'; - -describe('gitlab yaml visualization component', () => { - const defaultProps = { blobData: yamlString }; - let wrapper; - - const createComponent = props => { - return shallowMount(GitlabCiYamlVisualization, { - propsData: { - ...defaultProps, - ...props, - }, - }); - }; - - const findGlTabComponents = () => wrapper.findAll(GlTab); - const findPipelineGraph = () => wrapper.find(PipelineGraph); - - afterEach(() => { - wrapper.destroy(); - wrapper = null; - }); - - describe('tabs component', () => { - beforeEach(() => { - wrapper = createComponent(); - }); - - it('renders the file and visualization tabs', () => { - expect(findGlTabComponents()).toHaveLength(2); - }); - }); - - describe('graph component', () => { - beforeEach(() => { - wrapper = createComponent(); - }); - - it('is hidden by default', () => { - expect(findPipelineGraph().exists()).toBe(false); - }); - }); -}); diff --git a/spec/frontend/pipelines/pipeline_graph/utils_spec.js b/spec/frontend/pipelines/pipeline_graph/utils_spec.js index ade026c7053..b073ad7647e 100644 --- a/spec/frontend/pipelines/pipeline_graph/utils_spec.js +++ b/spec/frontend/pipelines/pipeline_graph/utils_spec.js @@ -1,11 +1,6 @@ -import { - preparePipelineGraphData, - createUniqueJobId, - generateJobNeedsDict, -} from '~/pipelines/utils'; +import { createUniqueJobId, generateJobNeedsDict } from '~/pipelines/utils'; describe('utils functions', () => { - const emptyResponse = { stages: [], jobs: {} }; const jobName1 = 'build_1'; const jobName2 = 'build_2'; const jobName3 = 'test_1'; @@ -66,115 +61,6 @@ describe('utils functions', () => { }, }; - describe('preparePipelineGraphData', () => { - describe('returns an empty array of stages and empty job objects if', () => { - it('no data is passed', () => { - expect(preparePipelineGraphData({})).toEqual(emptyResponse); - }); - - it('no stages are found', () => { - expect(preparePipelineGraphData({ includes: 'template/myTemplate.gitlab-ci.yml' })).toEqual( - emptyResponse, - ); - }); - }); - - describe('returns the correct array of stages and object of jobs', () => { - it('when multiple jobs are in the same stage', () => { - const expectedData = { - stages: [ - { - name: job1.stage, - groups: [ - { - name: jobName1, - jobs: [{ ...job1 }], - id: createUniqueJobId(job1.stage, jobName1), - }, - { - name: jobName2, - jobs: [{ ...job2 }], - id: createUniqueJobId(job2.stage, jobName2), - }, - ], - }, - ], - jobs: { - [jobName1]: { ...job1, id: createUniqueJobId(job1.stage, jobName1) }, - [jobName2]: { ...job2, id: createUniqueJobId(job2.stage, jobName2) }, - }, - }; - expect( - preparePipelineGraphData({ [jobName1]: { ...job1 }, [jobName2]: { ...job2 } }), - ).toEqual(expectedData); - }); - - it('when stages are defined by the user', () => { - const userDefinedStage2 = 'myStage2'; - - const expectedData = { - stages: [ - { - name: userDefinedStage, - groups: [], - }, - { - name: userDefinedStage2, - groups: [], - }, - ], - jobs: {}, - }; - - expect(preparePipelineGraphData({ stages: [userDefinedStage, userDefinedStage2] })).toEqual( - expectedData, - ); - }); - - it('by combining user defined stage and job stages, it preserves user defined order', () => { - const userDefinedStageThatOverlaps = 'deploy'; - - expect( - preparePipelineGraphData({ - stages: [userDefinedStage, userDefinedStageThatOverlaps], - [jobName1]: { ...job1 }, - [jobName2]: { ...job2 }, - [jobName3]: { ...job3 }, - [jobName4]: { ...job4 }, - }), - ).toEqual(pipelineGraphData); - }); - - it('with only unique values', () => { - const expectedData = { - stages: [ - { - name: job1.stage, - groups: [ - { - name: jobName1, - jobs: [{ ...job1 }], - id: createUniqueJobId(job1.stage, jobName1), - }, - ], - }, - ], - jobs: { - [jobName1]: { ...job1, id: createUniqueJobId(job1.stage, jobName1) }, - }, - }; - - expect( - preparePipelineGraphData({ - stages: ['build'], - [jobName1]: { ...job1 }, - [jobName1]: { ...job1 }, - }), - ).toEqual(expectedData); - }); - }); - }); - describe('generateJobNeedsDict', () => { it('generates an empty object if it receives no jobs', () => { expect(generateJobNeedsDict({ jobs: {} })).toEqual({}); diff --git a/spec/helpers/groups/group_members_helper_spec.rb b/spec/helpers/groups/group_members_helper_spec.rb index bb92445cb19..222cca43860 100644 --- a/spec/helpers/groups/group_members_helper_spec.rb +++ b/spec/helpers/groups/group_members_helper_spec.rb @@ -74,13 +74,15 @@ RSpec.describe Groups::GroupMembersHelper do before do allow(helper).to receive(:group_group_member_path).with(group, ':id').and_return('/groups/foo-bar/-/group_members/:id') + allow(helper).to receive(:can?).with(current_user, :admin_group_member, group).and_return(true) end it 'returns expected hash' do expect(helper.group_members_list_data_attributes(group, present_members([group_member]))).to include({ members: helper.members_data_json(group, present_members([group_member])), member_path: '/groups/foo-bar/-/group_members/:id', - group_id: group.id + group_id: group.id, + can_manage_members: 'true' }) end end diff --git a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb index 578743be96b..c8782eb3cec 100644 --- a/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/github_import/parallel_scheduling_spec.rb @@ -7,6 +7,10 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do Class.new do include(Gitlab::GithubImport::ParallelScheduling) + def importer_class + Class + end + def collection_method :issues end @@ -63,6 +67,82 @@ RSpec.describe Gitlab::GithubImport::ParallelScheduling do importer.execute end + + it 'logs the the process' do + importer = importer_class.new(project, client, parallel: false) + + expect(importer) + .to receive(:sequential_import) + .and_return([]) + + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting importer', + import_source: :github, + parallel: false, + project_id: project.id, + importer: 'Class' + ) + expect(logger) + .to receive(:info) + .with( + message: 'importer finished', + import_source: :github, + parallel: false, + project_id: project.id, + importer: 'Class' + ) + end + + importer.execute + end + + it 'logs the error when it fails' do + exception = StandardError.new('some error') + + importer = importer_class.new(project, client, parallel: false) + + expect(importer) + .to receive(:sequential_import) + .and_raise(exception) + + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting importer', + import_source: :github, + parallel: false, + project_id: project.id, + importer: 'Class' + ) + expect(logger) + .to receive(:error) + .with( + message: 'importer failed', + import_source: :github, + project_id: project.id, + parallel: false, + importer: 'Class', + 'error.message': 'some error' + ) + end + + expect(Gitlab::ErrorTracking) + .to receive(:track_and_raise_exception) + .with( + exception, + import_source: :github, + parallel: false, + project_id: project.id, + importer: 'Class' + ) + .and_call_original + + expect { importer.execute }.to raise_error(exception) + end end describe '#sequential_import' do diff --git a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb index d0cbc6b35e2..75f2c7922de 100644 --- a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb @@ -22,20 +22,21 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do end describe '#import' do - it 'imports the object' do - representation_class = double(:representation_class) - importer_class = double(:importer_class) - importer_instance = double(:importer_instance) - representation = double(:representation) - project = double(:project, full_path: 'foo/bar') - client = double(:client) - + let(:representation_class) { double(:representation_class) } + let(:importer_class) { double(:importer_class, name: 'klass_name') } + let(:importer_instance) { double(:importer_instance) } + let(:representation) { double(:representation) } + let(:project) { double(:project, full_path: 'foo/bar', id: 1) } + let(:client) { double(:client) } + + before do expect(worker) .to receive(:representation_class) .and_return(representation_class) expect(worker) .to receive(:importer_class) + .at_least(:once) .and_return(importer_class) expect(representation_class) @@ -47,7 +48,9 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do .to receive(:new) .with(representation, project, client) .and_return(importer_instance) + end + it 'imports the object' do expect(importer_instance) .to receive(:execute) @@ -55,8 +58,61 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do .to receive(:increment) .and_call_original + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting importer', + import_source: :github, + project_id: 1, + importer: 'klass_name' + ) + expect(logger) + .to receive(:info) + .with( + message: 'importer finished', + import_source: :github, + project_id: 1, + importer: 'klass_name' + ) + end + worker.import(project, client, { 'number' => 10 }) end + + it 'logs error when the import fails' do + exception = StandardError.new('some error') + expect(importer_instance) + .to receive(:execute) + .and_raise(exception) + + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting importer', + import_source: :github, + project_id: project.id, + importer: 'klass_name' + ) + expect(logger) + .to receive(:error) + .with( + message: 'importer failed', + import_source: :github, + project_id: project.id, + importer: 'klass_name', + 'error.message': 'some error' + ) + end + + expect(Gitlab::ErrorTracking) + .to receive(:track_and_raise_exception) + .with(exception, import_source: :github, project_id: 1, importer: 'klass_name') + .and_call_original + + expect { worker.import(project, client, { 'number' => 10 }) }.to raise_error(exception) + end end describe '#counter' do diff --git a/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb index 03e875bcb87..651ea77a71c 100644 --- a/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/stage_methods_spec.rb @@ -5,7 +5,13 @@ require 'spec_helper' RSpec.describe Gitlab::GithubImport::StageMethods do let(:project) { create(:project) } let(:worker) do - Class.new { include(Gitlab::GithubImport::StageMethods) }.new + Class.new do + def self.name + 'DummyStage' + end + + include(Gitlab::GithubImport::StageMethods) + end.new end describe '#perform' do @@ -30,8 +36,72 @@ RSpec.describe Gitlab::GithubImport::StageMethods do an_instance_of(Project) ) + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting stage', + import_source: :github, + project_id: project.id, + import_stage: 'DummyStage' + ) + expect(logger) + .to receive(:info) + .with( + message: 'stage finished', + import_source: :github, + project_id: project.id, + import_stage: 'DummyStage' + ) + end + worker.perform(project.id) end + + it 'logs error when import fails' do + exception = StandardError.new('some error') + + allow(worker) + .to receive(:find_project) + .with(project.id) + .and_return(project) + + expect(worker) + .to receive(:try_import) + .and_raise(exception) + + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'starting stage', + import_source: :github, + project_id: project.id, + import_stage: 'DummyStage' + ) + expect(logger) + .to receive(:error) + .with( + message: 'stage failed', + import_source: :github, + project_id: project.id, + import_stage: 'DummyStage', + 'error.message': 'some error' + ) + end + + expect(Gitlab::ErrorTracking) + .to receive(:track_and_raise_exception) + .with( + exception, + import_source: :github, + project_id: project.id, + import_stage: 'DummyStage' + ) + .and_call_original + + expect { worker.perform(project.id) }.to raise_error(exception) + end end describe '#try_import' do diff --git a/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb index 211eba993f7..4039cdac721 100644 --- a/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::GithubImport::ImportDiffNoteWorker do describe '#import' do it 'imports a diff note' do - project = double(:project, full_path: 'foo/bar') + project = double(:project, full_path: 'foo/bar', id: 1) client = double(:client) importer = double(:importer) hash = { diff --git a/spec/workers/gitlab/github_import/import_issue_worker_spec.rb b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb index 1d285790ee2..c25e89f6928 100644 --- a/spec/workers/gitlab/github_import/import_issue_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::GithubImport::ImportIssueWorker do describe '#import' do it 'imports an issue' do - project = double(:project, full_path: 'foo/bar') + project = double(:project, full_path: 'foo/bar', id: 1) client = double(:client) importer = double(:importer) hash = { diff --git a/spec/workers/gitlab/github_import/import_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_note_worker_spec.rb index 618aca4ff0a..bfb40d7c3d3 100644 --- a/spec/workers/gitlab/github_import/import_note_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_note_worker_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::GithubImport::ImportNoteWorker do describe '#import' do it 'imports a note' do - project = double(:project, full_path: 'foo/bar') + project = double(:project, full_path: 'foo/bar', id: 1) client = double(:client) importer = double(:importer) hash = { diff --git a/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb index 0f5df302d56..12b21abf910 100644 --- a/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::GithubImport::ImportPullRequestWorker do describe '#import' do it 'imports a pull request' do - project = double(:project, full_path: 'foo/bar') + project = double(:project, full_path: 'foo/bar', id: 1) client = double(:client) importer = double(:importer) hash = { diff --git a/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb b/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb index c821e0aaa09..2615da2be15 100644 --- a/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb +++ b/spec/workers/gitlab/github_import/stage/finish_import_worker_spec.rb @@ -26,7 +26,17 @@ RSpec.describe Gitlab::GithubImport::Stage::FinishImportWorker do .to receive(:increment) .and_call_original - expect(worker.logger).to receive(:info).with(an_instance_of(String)) + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + message: 'GitHub project import finished', + import_stage: 'Gitlab::GithubImport::Stage::FinishImportWorker', + import_source: :github, + project_id: project.id, + duration_s: a_kind_of(Numeric) + ) + end worker.report_import_time(project) end |