diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-07-12 21:08:58 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-07-12 21:08:58 +0300 |
commit | 31f59b55c63f6a7add79c5987731387ae3a4f7ab (patch) | |
tree | 3786af9493ae5634b35098a184993b2f134a5286 | |
parent | 8562dfae56800770e729dcb8215ebf7e1e29a55f (diff) |
Add latest changes from gitlab-org/gitlab@master
108 files changed, 2010 insertions, 435 deletions
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 28467dca3d5..66e4ab595b7 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -2154,6 +2154,8 @@ rules: # Requiring $DEPENDENCY_REVIEW_PAT prevents the bot from running on forks or CE # Without it the script would fail too. + - if: $ENABLE_DEPSCORE != 'true' + when: never - if: "$DEPENDENCY_REVIEW_PAT == null" when: never - <<: *if-not-ee diff --git a/.gitlab/ci/templates/gem.gitlab-ci.yml b/.gitlab/ci/templates/gem.gitlab-ci.yml index f1ee3426070..46c5e1342c6 100644 --- a/.gitlab/ci/templates/gem.gitlab-ci.yml +++ b/.gitlab/ci/templates/gem.gitlab-ci.yml @@ -11,7 +11,6 @@ spec: --- .gems:rules:$[[inputs.gem_name]]: rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"' - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached"' changes: - "$[[inputs.gem_path_prefix]]$[[inputs.gem_name]]/**/*" diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index 4b11d048bb9..a957f159f81 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -4991,7 +4991,6 @@ RSpec/MissingFeatureCategory: - 'spec/models/projects/project_topic_spec.rb' - 'spec/models/projects/repository_storage_move_spec.rb' - 'spec/models/projects/topic_spec.rb' - - 'spec/models/projects/triggered_hooks_spec.rb' - 'spec/models/projects/wiki_repository_spec.rb' - 'spec/models/prometheus_alert_event_spec.rb' - 'spec/models/prometheus_alert_spec.rb' diff --git a/app/assets/javascripts/contribution_events/components/contribution_event/contribution_event_private.vue b/app/assets/javascripts/contribution_events/components/contribution_event/contribution_event_private.vue new file mode 100644 index 00000000000..ba9bc25e310 --- /dev/null +++ b/app/assets/javascripts/contribution_events/components/contribution_event/contribution_event_private.vue @@ -0,0 +1,24 @@ +<script> +import { s__ } from '~/locale'; +import ContributionEventBase from './contribution_event_base.vue'; + +export default { + name: 'ContributionEventPrivate', + i18n: { + message: s__('ContributionEvent|Made a private contribution.'), + }, + components: { ContributionEventBase }, + props: { + event: { + type: Object, + required: true, + }, + }, +}; +</script> + +<template> + <contribution-event-base :event="event" icon-name="eye-slash">{{ + $options.i18n.message + }}</contribution-event-base> +</template> diff --git a/app/assets/javascripts/contribution_events/components/contribution_events.vue b/app/assets/javascripts/contribution_events/components/contribution_events.vue index 63bee40d0b3..eb28d058e4c 100644 --- a/app/assets/javascripts/contribution_events/components/contribution_events.vue +++ b/app/assets/javascripts/contribution_events/components/contribution_events.vue @@ -6,12 +6,14 @@ import { EVENT_TYPE_JOINED, EVENT_TYPE_LEFT, EVENT_TYPE_PUSHED, + EVENT_TYPE_PRIVATE, } from '../constants'; import ContributionEventApproved from './contribution_event/contribution_event_approved.vue'; import ContributionEventExpired from './contribution_event/contribution_event_expired.vue'; import ContributionEventJoined from './contribution_event/contribution_event_joined.vue'; import ContributionEventLeft from './contribution_event/contribution_event_left.vue'; import ContributionEventPushed from './contribution_event/contribution_event_pushed.vue'; +import ContributionEventPrivate from './contribution_event/contribution_event_private.vue'; export default { props: { @@ -121,6 +123,9 @@ export default { case EVENT_TYPE_PUSHED: return ContributionEventPushed; + case EVENT_TYPE_PRIVATE: + return ContributionEventPrivate; + default: return EmptyComponent; } diff --git a/app/assets/javascripts/profile/components/profile_tabs.vue b/app/assets/javascripts/profile/components/profile_tabs.vue index 3a30c3bdc9b..e24167eb4fa 100644 --- a/app/assets/javascripts/profile/components/profile_tabs.vue +++ b/app/assets/javascripts/profile/components/profile_tabs.vue @@ -5,6 +5,7 @@ import { getUserProjects } from '~/rest_api'; import { s__ } from '~/locale'; import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; import { createAlert } from '~/alert'; +import { VISIBILITY_LEVEL_PUBLIC_STRING } from '~/visibility_level/constants'; import OverviewTab from './overview_tab.vue'; import ActivityTab from './activity_tab.vue'; import GroupsTab from './groups_tab.vue'; @@ -81,7 +82,21 @@ export default { async mounted() { try { const response = await getUserProjects(this.userId, { per_page: 10 }); - this.personalProjects = convertObjectPropsToCamelCase(response.data, { deep: true }); + this.personalProjects = convertObjectPropsToCamelCase(response.data, { deep: true }).map( + (project) => { + // This API does not return the `visibility` key if user is signed out. + // Because this API only returns public projects when signed out, in this case, we can assume + // the `visibility` attribute is `public` if it is missing. + if (!project.visibility) { + return { + ...project, + visibility: VISIBILITY_LEVEL_PUBLIC_STRING, + }; + } + + return project; + }, + ); this.personalProjectsLoading = false; } catch (error) { createAlert({ message: this.$options.i18n.personalProjectsErrorMessage }); diff --git a/app/assets/javascripts/vue_shared/components/projects_list/projects_list_item.vue b/app/assets/javascripts/vue_shared/components/projects_list/projects_list_item.vue index 266cce29e50..a6535e4a514 100644 --- a/app/assets/javascripts/vue_shared/components/projects_list/projects_list_item.vue +++ b/app/assets/javascripts/vue_shared/components/projects_list/projects_list_item.vue @@ -85,11 +85,14 @@ export default { }; }, computed: { + visibility() { + return this.project.visibility; + }, visibilityIcon() { - return VISIBILITY_TYPE_ICON[this.project.visibility]; + return VISIBILITY_TYPE_ICON[this.visibility]; }, visibilityTooltip() { - return PROJECT_VISIBILITY_TYPE[this.project.visibility]; + return PROJECT_VISIBILITY_TYPE[this.visibility]; }, accessLevel() { return this.project.permissions?.projectAccess?.accessLevel; @@ -161,6 +164,7 @@ export default { > <template #meta> <gl-icon + v-if="visibility" v-gl-tooltip="visibilityTooltip" :name="visibilityIcon" class="gl-text-secondary gl-ml-3" @@ -248,7 +252,10 @@ export default { <span>{{ numberToMetricPrefix(project.openIssuesCount) }}</span> </gl-link> </div> - <div class="gl-font-sm gl-white-space-nowrap gl-text-secondary gl-mt-3"> + <div + v-if="project.updatedAt" + class="gl-font-sm gl-white-space-nowrap gl-text-secondary gl-mt-3" + > <span>{{ $options.i18n.updated }}</span> <time-ago-tooltip :time="project.updatedAt" /> </div> diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 3a3efdef95f..ebc43b04b1b 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -85,4 +85,8 @@ class AwardEmoji < ApplicationRecord def to_ability_name 'emoji' end + + def hook_attrs + Gitlab::HookData::EmojiBuilder.new(self).build + end end diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index c2d7529f468..fde528e3fa0 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -58,6 +58,10 @@ class BulkImport < ApplicationRecord Gitlab::VersionInfo.new(MIN_MAJOR_VERSION, MIN_MINOR_VERSION_FOR_PROJECT) end + def self.min_gl_version_for_migration_in_batches + Gitlab::VersionInfo.new(16, 2) + end + def self.all_human_statuses state_machine.states.map(&:human_name) end @@ -68,4 +72,8 @@ class BulkImport < ApplicationRecord update!(has_failures: true) end + + def supports_batched_export? + source_version_info >= self.class.min_gl_version_for_migration_in_batches + end end diff --git a/app/models/bulk_imports/batch_tracker.rb b/app/models/bulk_imports/batch_tracker.rb index df1fab89ee6..2e79d41d46e 100644 --- a/app/models/bulk_imports/batch_tracker.rb +++ b/app/models/bulk_imports/batch_tracker.rb @@ -25,9 +25,7 @@ module BulkImports end event :finish do - transition started: :finished - transition failed: :failed - transition skipped: :skipped + transition any => :finished end event :skip do diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 94e4a8165eb..4f50a112141 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -144,12 +144,27 @@ class BulkImports::Entity < ApplicationRecord end end - def export_relations_url_path - "#{base_resource_path}/export_relations" + def export_relations_url_path_base + File.join(base_resource_path, 'export_relations') end - def relation_download_url_path(relation) - "#{export_relations_url_path}/download?relation=#{relation}" + def export_relations_url_path(batched: false) + if batched && bulk_import.supports_batched_export? + Gitlab::Utils.add_url_parameters(export_relations_url_path_base, batched: batched) + else + export_relations_url_path_base + end + end + + def relation_download_url_path(relation, batch_number = nil) + url = File.join(export_relations_url_path_base, 'download') + params = { relation: relation } + + if batch_number && bulk_import.supports_batched_export? + params.merge!(batched: true, batch_number: batch_number) + end + + Gitlab::Utils.add_url_parameters(url, params) end def wikis_url_path diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 93cf047c690..5c3f8e4b8d4 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -32,9 +32,7 @@ module BulkImports end event :finish do - transition started: :finished - transition finished: :finished - transition failed: :failed + transition any => :finished end event :fail_op do @@ -63,5 +61,12 @@ module BulkImports FileTransfer.config_for(portable) end end + + def remove_existing_upload! + return unless upload&.export_file&.file + + upload.remove_export_file! + upload.save! + end end end diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index cbd7b189007..3d820e65d5b 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -13,28 +13,48 @@ module BulkImports end def started? - !empty? && export_status['status'] == Export::STARTED + !empty? && status['status'] == Export::STARTED end def failed? - !empty? && export_status['status'] == Export::FAILED + !empty? && status['status'] == Export::FAILED end def empty? - export_status.nil? + status.nil? end def error - export_status['error'] + status['error'] + end + + def batched? + status['batched'] == true + end + + def batches_count + status['batches_count'].to_i + end + + def batch(batch_number) + raise ArgumentError if batch_number < 1 + + return unless batched? + + status['batches'].find { |item| item['batch_number'] == batch_number } end private attr_reader :client, :entity, :relation, :pipeline_tracker - def export_status - strong_memoize(:export_status) do - fetch_export_status&.find { |item| item['relation'] == relation } + def status + strong_memoize(:status) do + status = fetch_status + + next status if status.is_a?(Hash) || status.nil? + + status.find { |item| item['relation'] == relation } rescue BulkImports::NetworkError => e raise BulkImports::RetryPipelineError.new(e.message, 2.seconds) if e.retriable?(pipeline_tracker) @@ -44,12 +64,12 @@ module BulkImports end end - def fetch_export_status - client.get(status_endpoint).parsed_response + def fetch_status + client.get(status_endpoint, relation: relation).parsed_response end def status_endpoint - File.join(entity.export_relations_url_path, 'status') + File.join(entity.export_relations_url_path_base, 'status') end def default_error_response(message) diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 55502721a76..d1a6f3b9a80 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -24,6 +24,7 @@ class BulkImports::Tracker < ApplicationRecord delegate :file_extraction_pipeline?, to: :pipeline_class DEFAULT_PAGE_SIZE = 500 + STALE_AFTER = 4.hours scope :next_pipeline_trackers_for, -> (entity_id) { entity_scope = where(bulk_import_entity_id: entity_id) @@ -89,4 +90,8 @@ class BulkImports::Tracker < ApplicationRecord transition [:created, :started] => :timeout end end + + def stale? + created_at < STALE_AFTER.ago + end end diff --git a/app/models/commit.rb b/app/models/commit.rb index 26412205899..ded4b06a028 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -149,6 +149,10 @@ class Commit from_hash(hash, project) end + + def underscore + 'commit' + end end attr_accessor :raw diff --git a/app/models/concerns/triggerable_hooks.rb b/app/models/concerns/triggerable_hooks.rb index e3800caa43f..0e72bd30a37 100644 --- a/app/models/concerns/triggerable_hooks.rb +++ b/app/models/concerns/triggerable_hooks.rb @@ -17,7 +17,8 @@ module TriggerableHooks feature_flag_hooks: :feature_flag_events, release_hooks: :releases_events, member_hooks: :member_events, - subgroup_hooks: :subgroup_events + subgroup_hooks: :subgroup_events, + emoji_hooks: :emoji_events }.freeze extend ActiveSupport::Concern diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 695041f0247..05c5ad22218 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -21,7 +21,8 @@ class ProjectHook < WebHook :wiki_page_hooks, :deployment_hooks, :feature_flag_hooks, - :release_hooks + :release_hooks, + :emoji_hooks ] belongs_to :project diff --git a/app/models/issue.rb b/app/models/issue.rb index f94ebf30de3..4ac9ff6f3b5 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -754,6 +754,10 @@ class Issue < ApplicationRecord issue_email_participants.find_by_email(email)&.destroy end + def hook_attrs + Gitlab::HookData::IssueBuilder.new(self).build + end + private def check_issue_type_in_sync! diff --git a/app/models/projects/triggered_hooks.rb b/app/models/projects/triggered_hooks.rb index e3aa3d106b7..1f51ced5b57 100644 --- a/app/models/projects/triggered_hooks.rb +++ b/app/models/projects/triggered_hooks.rb @@ -17,6 +17,8 @@ module Projects # Assumes that the relations implement TriggerableHooks @relations.each do |hooks| hooks.hooks_for(@scope).select_active(@scope, @data).each do |hook| + next if @scope == :emoji_hooks && Feature.disabled?(:emoji_webhooks, hook.parent) + hook.async_execute(@data, @scope.to_s) end end diff --git a/app/services/award_emojis/add_service.rb b/app/services/award_emojis/add_service.rb index f45a4330c09..065ef9dc708 100644 --- a/app/services/award_emojis/add_service.rb +++ b/app/services/award_emojis/add_service.rb @@ -27,6 +27,8 @@ module AwardEmojis def after_create(award) TodoService.new.new_award_emoji(todoable, current_user) if todoable + + execute_hooks(award, 'award') end def todoable diff --git a/app/services/award_emojis/base_service.rb b/app/services/award_emojis/base_service.rb index 626e26d63b5..274c528acf2 100644 --- a/app/services/award_emojis/base_service.rb +++ b/app/services/award_emojis/base_service.rb @@ -11,6 +11,13 @@ module AwardEmojis super(awardable.project, current_user) end + def execute_hooks(award_emoji, action) + return unless awardable.project&.has_active_hooks?(:emoji_hooks) + + hook_data = Gitlab::DataBuilder::Emoji.build(award_emoji, current_user, action) + awardable.project.execute_hooks(hook_data, :emoji_hooks) + end + private def normalize_name(name) diff --git a/app/services/award_emojis/destroy_service.rb b/app/services/award_emojis/destroy_service.rb index 47dc8418e07..b7146d69bf0 100644 --- a/app/services/award_emojis/destroy_service.rb +++ b/app/services/award_emojis/destroy_service.rb @@ -22,6 +22,7 @@ module AwardEmojis private def after_destroy(award) + execute_hooks(award, 'revoke') end end end diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb index 142bc48efe3..ed71c09420b 100644 --- a/app/services/bulk_imports/relation_export_service.rb +++ b/app/services/bulk_imports/relation_export_service.rb @@ -16,7 +16,7 @@ module BulkImports def execute find_or_create_export! do |export| - remove_existing_export_file!(export) + export.remove_existing_upload! export_service.execute compress_exported_relation upload_compressed_file(export) @@ -45,15 +45,6 @@ module BulkImports fail_export!(export, e) end - def remove_existing_export_file!(export) - upload = export.upload - - return unless upload&.export_file&.file - - upload.remove_export_file! - upload.save! - end - def export_service @export_service ||= if config.tree_relation?(relation) || config.self_relation?(relation) TreeExportService.new(portable, export_path, relation, user) diff --git a/app/services/concerns/integrations/project_test_data.rb b/app/services/concerns/integrations/project_test_data.rb index b3427697052..fcef22a8cab 100644 --- a/app/services/concerns/integrations/project_test_data.rb +++ b/app/services/concerns/integrations/project_test_data.rb @@ -77,5 +77,20 @@ module Integrations release.to_hook_data('create') end + + def emoji_events_data + no_data_error(s_('TestHooks|Ensure the project has notes.')) unless project.notes.any? + + award_emoji = AwardEmoji.new( + id: 1, + name: 'thumbsup', + user: current_user, + awardable: project.notes.last, + created_at: Time.zone.now, + updated_at: Time.zone.now + ) + + Gitlab::DataBuilder::Emoji.build(award_emoji, current_user, 'award') + end end end diff --git a/app/services/integrations/test/project_service.rb b/app/services/integrations/test/project_service.rb index 31c8f02c7b6..48240f297fe 100644 --- a/app/services/integrations/test/project_service.rb +++ b/app/services/integrations/test/project_service.rb @@ -35,6 +35,8 @@ module Integrations deployment_events_data when 'release' releases_events_data + when 'award_emoji' + emoji_events_data end end end diff --git a/app/services/test_hooks/project_service.rb b/app/services/test_hooks/project_service.rb index dcd92ac2b8c..42af65ebd57 100644 --- a/app/services/test_hooks/project_service.rb +++ b/app/services/test_hooks/project_service.rb @@ -32,6 +32,8 @@ module TestHooks wiki_page_events_data when 'releases_events' releases_events_data + when 'emoji_events' + emoji_events_data end end end diff --git a/app/views/projects/settings/ci_cd/show.html.haml b/app/views/projects/settings/ci_cd/show.html.haml index c7bb6a7f5da..007169809c9 100644 --- a/app/views/projects/settings/ci_cd/show.html.haml +++ b/app/views/projects/settings/ci_cd/show.html.haml @@ -33,12 +33,13 @@ = render_if_exists 'projects/settings/ci_cd/protected_environments', expanded: expanded -%section.settings.no-animate#js-runners-settings{ class: ('expanded' if expanded || params[:expand_runners]), data: { qa_selector: 'runners_settings_content' } } +- expand_runners = expanded || params[:expand_runners] +%section.settings.no-animate#js-runners-settings{ class: ('expanded' if expand_runners), data: { qa_selector: 'runners_settings_content' } } .settings-header %h4.settings-title.js-settings-toggle.js-settings-toggle-trigger-only = _("Runners") = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle' }) do - = expanded ? _('Collapse') : _('Expand') + = expand_runners ? _('Collapse') : _('Expand') %p = _("Runners are processes that pick up and execute CI/CD jobs for GitLab.") = link_to s_('What is GitLab Runner?'), 'https://docs.gitlab.com/runner/', target: '_blank', rel: 'noopener noreferrer' diff --git a/app/views/shared/web_hooks/_form.html.haml b/app/views/shared/web_hooks/_form.html.haml index da6ce1652c4..4ca11eb3849 100644 --- a/app/views/shared/web_hooks/_form.html.haml +++ b/app/views/shared/web_hooks/_form.html.haml @@ -64,6 +64,13 @@ = form.gitlab_ui_checkbox_component :releases_events, integration_webhook_event_human_name(:releases_events), help_text: s_('Webhooks|A release is created or updated.') + - if Feature.enabled?(:emoji_webhooks, hook.parent) + %li.gl-pb-5 + - emoji_help_link = link_to s_('Which emoji events trigger webhooks'), help_page_path('user/project/integrations/webhook_events.md', anchor: 'emoji-events') + = form.gitlab_ui_checkbox_component :emoji_events, + integration_webhook_event_human_name(:emoji_events), + help_text: s_('Webhooks|An emoji is awarded or revoked. %{help_link}?').html_safe % { help_link: emoji_help_link } + .form-group = form.label :enable_ssl_verification, s_('Webhooks|SSL verification'), class: 'label-bold checkbox' %ul.list-unstyled diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index cbb6743755b..9e341fb9263 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2406,6 +2406,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: bulk_imports_finish_batched_pipeline + :worker_name: BulkImports::FinishBatchedPipelineWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: bulk_imports_finish_batched_relation_export :worker_name: BulkImports::FinishBatchedRelationExportWorker :feature_category: :importers @@ -2424,6 +2433,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: bulk_imports_pipeline_batch + :worker_name: BulkImports::PipelineBatchWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] - :name: bulk_imports_relation_batch_export :worker_name: BulkImports::RelationBatchExportWorker :feature_category: :importers diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb index 530419dac26..44759916f99 100644 --- a/app/workers/bulk_imports/export_request_worker.rb +++ b/app/workers/bulk_imports/export_request_worker.rb @@ -15,35 +15,40 @@ module BulkImports end def perform(entity_id) - entity = BulkImports::Entity.find(entity_id) + @entity = BulkImports::Entity.find(entity_id) - entity.update!(source_xid: entity_source_xid(entity)) if entity.source_xid.nil? - - request_export(entity) + set_source_xid + request_export BulkImports::EntityWorker.perform_async(entity_id) end def perform_failure(exception, entity_id) - entity = BulkImports::Entity.find(entity_id) + @entity = BulkImports::Entity.find(entity_id) - log_and_fail(exception, entity) + log_and_fail(exception) end private - def request_export(entity) - http_client(entity).post(entity.export_relations_url_path) + attr_reader :entity + + def set_source_xid + entity.update!(source_xid: entity_source_xid) if entity.source_xid.nil? + end + + def request_export + http_client.post(export_url) end - def http_client(entity) + def http_client @client ||= Clients::HTTP.new( url: entity.bulk_import.configuration.url, token: entity.bulk_import.configuration.access_token ) end - def failure_attributes(exception, entity) + def failure_attributes(exception) { bulk_import_entity_id: entity.id, pipeline_class: 'ExportRequestWorker', @@ -53,23 +58,20 @@ module BulkImports } end - def graphql_client(entity) + def graphql_client @graphql_client ||= BulkImports::Clients::Graphql.new( url: entity.bulk_import.configuration.url, token: entity.bulk_import.configuration.access_token ) end - def entity_source_xid(entity) - query = entity_query(entity) - client = graphql_client(entity) - - response = client.execute( - client.parse(query.to_s), + def entity_source_xid + response = graphql_client.execute( + graphql_client.parse(entity_query.to_s), { full_path: entity.source_full_path } ).original_hash - ::GlobalID.parse(response.dig(*query.data_path, 'id')).model_id + ::GlobalID.parse(response.dig(*entity_query.data_path, 'id')).model_id rescue StandardError => e log_exception(e, { @@ -86,12 +88,12 @@ module BulkImports nil end - def entity_query(entity) - if entity.group? - BulkImports::Groups::Graphql::GetGroupQuery.new(context: nil) - else - BulkImports::Projects::Graphql::GetProjectQuery.new(context: nil) - end + def entity_query + @entity_query ||= if entity.group? + BulkImports::Groups::Graphql::GetGroupQuery.new(context: nil) + else + BulkImports::Projects::Graphql::GetProjectQuery.new(context: nil) + end end def logger @@ -104,7 +106,7 @@ module BulkImports logger.error(structured_payload(payload)) end - def log_and_fail(exception, entity) + def log_and_fail(exception) log_exception(exception, { bulk_import_entity_id: entity.id, @@ -117,9 +119,13 @@ module BulkImports } ) - BulkImports::Failure.create(failure_attributes(exception, entity)) + BulkImports::Failure.create(failure_attributes(exception)) entity.fail_op! end + + def export_url + entity.export_relations_url_path(batched: Feature.enabled?(:bulk_imports_batched_import_export)) + end end end diff --git a/app/workers/bulk_imports/finish_batched_pipeline_worker.rb b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb new file mode 100644 index 00000000000..4200d0e4a0f --- /dev/null +++ b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module BulkImports + class FinishBatchedPipelineWorker + include ApplicationWorker + include ExceptionBacktrace + + REQUEUE_DELAY = 5.seconds + + idempotent! + deduplicate :until_executing + data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency + feature_category :importers + + def perform(pipeline_tracker_id) + @tracker = Tracker.find(pipeline_tracker_id) + + return unless tracker.batched? + return unless tracker.started? + return re_enqueue if import_in_progress? + + if tracker.stale? + tracker.batches.map(&:fail_op!) + tracker.fail_op! + else + tracker.finish! + end + + ensure + ::BulkImports::EntityWorker.perform_async(tracker.entity.id, tracker.stage) + end + + private + + attr_reader :tracker + + def re_enqueue + self.class.perform_in(REQUEUE_DELAY, tracker.id) + end + + def import_in_progress? + tracker.batches.any?(&:started?) + end + end +end diff --git a/app/workers/bulk_imports/finish_batched_relation_export_worker.rb b/app/workers/bulk_imports/finish_batched_relation_export_worker.rb index aa7bbffa732..92a33a971e7 100644 --- a/app/workers/bulk_imports/finish_batched_relation_export_worker.rb +++ b/app/workers/bulk_imports/finish_batched_relation_export_worker.rb @@ -5,7 +5,7 @@ module BulkImports include ApplicationWorker idempotent! - data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency + data_consistency :sticky feature_category :importers REENQUEUE_DELAY = 5.seconds diff --git a/app/workers/bulk_imports/pipeline_batch_worker.rb b/app/workers/bulk_imports/pipeline_batch_worker.rb new file mode 100644 index 00000000000..378eff99b52 --- /dev/null +++ b/app/workers/bulk_imports/pipeline_batch_worker.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module BulkImports + class PipelineBatchWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency + feature_category :importers + sidekiq_options retry: false, dead: false + worker_has_external_dependencies! + + def perform(batch_id) + @batch = ::BulkImports::BatchTracker.find(batch_id) + @tracker = @batch.tracker + + try_obtain_lease { run } + ensure + ::BulkImports::FinishBatchedPipelineWorker.perform_async(tracker.id) + end + + private + + attr_reader :batch, :tracker + + def run + return batch.skip! if tracker.failed? || tracker.finished? + + batch.start! + tracker.pipeline_class.new(context).run + batch.finish! + rescue BulkImports::RetryPipelineError => e + retry_batch(e) + rescue StandardError => e + fail_batch(e) + end + + def fail_batch(exception) + batch.fail_op! + + Gitlab::ErrorTracking.track_exception( + exception, + batch_id: batch.id, + tracker_id: tracker.id, + pipeline_class: tracker.pipeline_name, + pipeline_step: 'pipeline_batch_worker_run' + ) + + BulkImports::Failure.create( + bulk_import_entity_id: batch.tracker.entity.id, + pipeline_class: tracker.pipeline_name, + pipeline_step: 'pipeline_batch_worker_run', + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + ) + end + + def context + @context ||= ::BulkImports::Pipeline::Context.new(tracker, batch_number: batch.batch_number) + end + + def retry_batch(exception) + batch.retry! + + re_enqueue(exception.retry_delay) + end + + def lease_timeout + 30 + end + + def lease_key + "gitlab:bulk_imports:pipeline_batch_worker:#{batch.id}" + end + + def re_enqueue(delay = FILE_EXTRACTION_PIPELINE_PERFORM_DELAY) + self.class.perform_in(delay, batch.id) + end + end +end diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index f03e0bc0656..e0db18cb987 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -31,7 +31,6 @@ module BulkImports fail_tracker(StandardError.new(message)) unless pipeline_tracker.finished? || pipeline_tracker.skipped? end end - ensure ::BulkImports::EntityWorker.perform_async(entity_id, stage) end @@ -49,9 +48,17 @@ module BulkImports return re_enqueue if export_empty? || export_started? - pipeline_tracker.update!(status_event: 'start', jid: jid) - pipeline_tracker.pipeline_class.new(context).run - pipeline_tracker.finish! + if file_extraction_pipeline? && export_status.batched? + pipeline_tracker.update!(status_event: 'start', jid: jid, batched: true) + + return pipeline_tracker.finish! if export_status.batches_count < 1 + + enqueue_batches + else + pipeline_tracker.update!(status_event: 'start', jid: jid) + pipeline_tracker.pipeline_class.new(context).run + pipeline_tracker.finish! + end rescue BulkImports::RetryPipelineError => e retry_tracker(e) rescue StandardError => e @@ -179,5 +186,13 @@ module BulkImports time_since_tracker_created > Pipeline::NDJSON_EXPORT_TIMEOUT end + + def enqueue_batches + 1.upto(export_status.batches_count) do |batch_number| + batch = pipeline_tracker.batches.find_or_create_by!(batch_number: batch_number) # rubocop:disable CodeReuse/ActiveRecord + + ::BulkImports::PipelineBatchWorker.perform_async(batch.id) + end + end end end diff --git a/config/feature_flags/development/bulk_imports_batched_import_export.yml b/config/feature_flags/development/bulk_imports_batched_import_export.yml new file mode 100644 index 00000000000..4afb715b1ee --- /dev/null +++ b/config/feature_flags/development/bulk_imports_batched_import_export.yml @@ -0,0 +1,8 @@ +--- +name: bulk_imports_batched_import_export +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124434 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/406559 +milestone: '16.2' +type: development +group: group::import and integrate +default_enabled: false diff --git a/config/feature_flags/development/emoji_webhooks.yml b/config/feature_flags/development/emoji_webhooks.yml new file mode 100644 index 00000000000..98d1918d365 --- /dev/null +++ b/config/feature_flags/development/emoji_webhooks.yml @@ -0,0 +1,8 @@ +--- +name: emoji_webhooks +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123952 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/417288 +milestone: '16.2' +type: development +group: group::import and integrate +default_enabled: false diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 18ae1d7c097..48805bc7e78 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -101,10 +101,14 @@ - 1 - - bulk_imports_export_request - 1 +- - bulk_imports_finish_batched_pipeline + - 1 - - bulk_imports_finish_batched_relation_export - 1 - - bulk_imports_pipeline - 1 +- - bulk_imports_pipeline_batch + - 1 - - bulk_imports_relation_batch_export - 1 - - bulk_imports_relation_export diff --git a/db/migrate/20230710200434_add_emoji_events_to_web_hooks.rb b/db/migrate/20230710200434_add_emoji_events_to_web_hooks.rb new file mode 100644 index 00000000000..45a6d15cefb --- /dev/null +++ b/db/migrate/20230710200434_add_emoji_events_to_web_hooks.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddEmojiEventsToWebHooks < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def change + add_column :web_hooks, :emoji_events, :boolean, null: false, default: false + end +end diff --git a/db/post_migrate/20230707220646_add_index_to_vulnerability_findings_on_uuid_again.rb b/db/post_migrate/20230707220646_add_index_to_vulnerability_findings_on_uuid_again.rb new file mode 100644 index 00000000000..06d0117d50e --- /dev/null +++ b/db/post_migrate/20230707220646_add_index_to_vulnerability_findings_on_uuid_again.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIndexToVulnerabilityFindingsOnUuidAgain < Gitlab::Database::Migration[2.1] + INDEX_NAME = 'index_vuln_findings_on_uuid_including_vuln_id' + + def up + Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.where(name: INDEX_NAME).find_each do |record| + record.definition = record.definition.strip + record.save! + end + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20230707220646 b/db/schema_migrations/20230707220646 new file mode 100644 index 00000000000..7577fce9131 --- /dev/null +++ b/db/schema_migrations/20230707220646 @@ -0,0 +1 @@ +6f801df7ed92d70e6f603d36b8c23c4e133e09b05040f848ca8d71581b8a793f
\ No newline at end of file diff --git a/db/schema_migrations/20230710200434 b/db/schema_migrations/20230710200434 new file mode 100644 index 00000000000..25bedb60700 --- /dev/null +++ b/db/schema_migrations/20230710200434 @@ -0,0 +1 @@ +a2ee4cf351b3befa61e6631d6e7131f01c00cb45dcc96968840e809d9b32fae0
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7db5bfbca1e..0798515b5f4 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -24658,7 +24658,8 @@ CREATE TABLE web_hooks ( encrypted_url_variables bytea, encrypted_url_variables_iv bytea, integration_id integer, - branch_filter_strategy smallint DEFAULT 0 NOT NULL + branch_filter_strategy smallint DEFAULT 0 NOT NULL, + emoji_events boolean DEFAULT false NOT NULL ); CREATE SEQUENCE web_hooks_id_seq diff --git a/doc/administration/merge_requests_approvals.md b/doc/administration/merge_requests_approvals.md new file mode 100644 index 00000000000..6cd0edf22eb --- /dev/null +++ b/doc/administration/merge_requests_approvals.md @@ -0,0 +1,43 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, concepts +--- + +# Merge request approvals **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39060) in GitLab 12.8. + +Merge request approval rules prevent users from overriding certain settings on the project level. +When enabled at the instance level, these settings [cascade](../user/project/merge_requests/approvals/settings.md#settings-cascading) +and can no longer be changed: + +- In projects. +- In groups. Cascading to groups was [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) + in GitLab 14.5. + +To enable merge request approval settings for an instance: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. Select **Push Rules**. +1. Expand **Merge request approvals**. +1. Choose the required options. +1. Select **Save changes**. + +## Available rules + +Merge request approval settings that can be set at an instance level are: + +- **Prevent approval by author**. Prevents project maintainers from allowing request authors to + merge their own merge requests. +- **Prevent approvals by users who add commits**. Prevents project maintainers from allowing users + to approve merge requests if they have submitted any commits to the source branch. +- **Prevent editing approval rules in projects and merge requests**. Prevents users from modifying + the approvers list in project settings or in individual merge requests. + +See also the following, which are affected by instance-level rules: + +- [Project merge request approval rules](../user/project/merge_requests/approvals/index.md). +- [Group merge request approval settings](../user/group/manage.md#group-merge-request-approval-settings) available in GitLab 13.9 and later. diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index e3867a25846..3c392e60ed3 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -1007,28 +1007,27 @@ NOTE: Retention policies in your object storage provider, such as Amazon S3 Lifecycle, may prevent objects from being properly deleted. -Container Registry can use considerable amounts of disk space. To clear up -some unused layers, the registry includes a garbage collect command. +The container registry can use considerable amounts of storage space, and you might want to +[reduce storage usage](../../user/packages/container_registry/reduce_container_registry_storage.md). +Among the listed options, deleting tags is the most effective option. However, tag deletion +alone does not delete image layers, it only leaves the underlying image manifests untagged. -GitLab offers a set of APIs to manipulate the Container Registry and aid the process -of removing unused tags. Currently, this is exposed using the API, but in the future, -these controls should migrate to the GitLab interface. +To more effectively free up space, the Container Registry has a garbage collector that can +delete unreferenced layers and (optionally) untagged manifests. -Users who have the [Maintainer role](../../user/permissions.md) for the project can -[delete Container Registry tags in bulk](../../api/container_registry.md#delete-registry-repository-tags-in-bulk) -periodically based on their own criteria. However, deleting the tags alone does not recycle data, -it only unlinks tags from manifests and image blobs. To recycle the Container -Registry data in the whole GitLab instance, you can use the built-in garbage collection command -provided by `gitlab-ctl`. +To start the garbage collector, use the `registry-garbage-collect` command provided by `gitlab-ctl`. + +WARNING: +This command shuts down the Container Registry prior to the garbage collection and +only starts it again after garbage collection completes. If you prefer to avoid downtime, +you can manually set the Container Registry to [read-only mode and bypass `gitlab-ctl`](#performing-garbage-collection-without-downtime). + +The time required to perform garbage collection is proportional to the Container Registry data size. Prerequisites: - You must have installed GitLab by using an Omnibus package or the [GitLab Helm chart](https://docs.gitlab.com/charts/charts/registry/#garbage-collection). -- You must set the Registry to [read-only mode](#performing-garbage-collection-without-downtime). - Running garbage collection causes downtime for the Container Registry. When you run this command - on an instance in an environment where another instance is still writing to the Registry storage, - referenced manifests are removed. ### Understanding the content-addressable layers @@ -1053,16 +1052,11 @@ Due to the architecture of registry, this data is still accessible when pulling image `my.registry.com/my.group/my.project@sha256:111111`, though it is no longer directly accessible via the `:latest` tag. -### Recycling unused tags +### Remove unreferenced layers -Before you run the built-in command, note the following: - -- The built-in command stops the registry before it starts the garbage collection. -- The garbage collect command takes some time to complete, depending on the - amount of data that exists. -- If you changed the location of registry configuration file, you must - specify its path. -- After the garbage collection is done, the registry should start automatically. +Image layers are the bulk of the Container Registry storage. A layer is considered +unreferenced when no image manifest references it. Unreferenced layers are the +default target of the Container Registry garbage collector. If you did not change the default location of the configuration file, run: @@ -1070,51 +1064,37 @@ If you did not change the default location of the configuration file, run: sudo gitlab-ctl registry-garbage-collect ``` -This command takes some time to complete, depending on the amount of -layers you have stored. - If you changed the location of the Container Registry `config.yml`: ```shell sudo gitlab-ctl registry-garbage-collect /path/to/config.yml ``` -You may also [remove all untagged manifests and unreferenced layers](#removing-untagged-manifests-and-unreferenced-layers), -although this is a way more destructive operation, and you should first -understand the implications. +You can also [remove all untagged manifests and unreferenced layers](#removing-untagged-manifests-and-unreferenced-layers) +to recover additional space. ### Removing untagged manifests and unreferenced layers -WARNING: -This is a destructive operation. +By default the Container Registry garbage collector ignores images that are untagged, +and users can keep pulling untagged images by digest. Users can also re-tag images +in the future, making them visible again in the GitLab UI and API. -The GitLab Container Registry follows the same default workflow as Docker Distribution: -retain untagged manifests and all layers, even ones that are not referenced directly. All content -can be accessed by using context addressable identifiers. - -However, in most workflows, you don't care about untagged manifests and old layers if they are not directly -referenced by a tagged manifest. The `registry-garbage-collect` command supports the -`-m` switch to allow you to remove all unreferenced manifests and layers that are -not directly accessible via `tag`: +If you do not care about untagged images and the layers exclusively referenced by these images, +you can delete them all. Use the `-m` flag on the `registry-garbage-collect` command: ```shell sudo gitlab-ctl registry-garbage-collect -m ``` -Since this is a way more destructive operation, this behavior is disabled by default. -You are likely expecting this way of operation, but before doing that, ensure -that you have backed up all registry data. - -When the command is used without the `-m` flag, the Container Registry only removes layers that are not referenced by any manifest, tagged or not. +If you are unsure about deleting untagged images, back up your registry data before proceeding. ### Performing garbage collection without downtime -You can perform garbage collection without stopping the Container Registry by putting -it in read-only mode and by not using the built-in command. On large instances -this could require Container Registry to be in read-only mode for a while. -During this time, -you are able to pull from the Container Registry, but you are not able to -push. +To do garbage collection while keeping the Container Registry online, put the registry +in read-only mode and bypass the built-in `gitlab-ctl registry-garbage-collect` command. + +You can pull but not push images while the Container Registry is in read-only mode. The Container +Registry must remain in read-only for the full duration of the garbage collection. By default, the [registry storage path](#configure-storage-for-the-container-registry) is `/var/opt/gitlab/gitlab-rails/shared/registry`. @@ -1146,18 +1126,15 @@ To enable the read-only mode: 1. Next, trigger one of the garbage collect commands: - WARNING: - You must use `/opt/gitlab/embedded/bin/registry` to recycle unused tags. If you use `gitlab-ctl registry-garbage-collect`, **the container registry goes down**. - ```shell - # Recycling unused tags + # Remove unreferenced layers sudo /opt/gitlab/embedded/bin/registry garbage-collect /var/opt/gitlab/registry/config.yml - # Removing unused layers not referenced by manifests + # Remove untagged manifests and unreferenced layers sudo /opt/gitlab/embedded/bin/registry garbage-collect -m /var/opt/gitlab/registry/config.yml ``` - This command starts the garbage collection, which might take some time to complete. + This command starts the garbage collection. The time to complete is proportional to the registry data size. 1. Once done, in `/etc/gitlab/gitlab.rb` change it back to read-write mode: diff --git a/doc/administration/settings/protected_paths.md b/doc/administration/settings/protected_paths.md new file mode 100644 index 00000000000..28ca3651387 --- /dev/null +++ b/doc/administration/settings/protected_paths.md @@ -0,0 +1,43 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference +--- + +# Protected paths **(FREE SELF)** + +Rate limiting is a technique that improves the security and durability of a web +application. For more details, see [Rate limits](../../security/rate_limits.md). + +You can rate limit (protect) specified paths. For these paths, GitLab responds with HTTP status +code `429` to POST requests at protected paths that exceed 10 requests per minute per IP address. + +For example, the following are limited to a maximum 10 requests per minute: + +- User sign-in +- User sign-up (if enabled) +- User password reset + +After 10 requests, the client must wait 60 seconds before it can try again. + +See also: + +- List of paths [protected by default](../instance_limits.md#by-protected-path). +- [User and IP rate limits](../../user/admin_area/settings/user_and_ip_rate_limits.md#response-headers) + for the headers returned to blocked requests. + +## Configure protected paths + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31246) in GitLab 12.4. + +Throttling of protected paths is enabled by default and can be disabled or +customized on **Admin > Network > Protected Paths**, along with these options: + +- Maximum number of requests per period per user. +- Rate limit period in seconds. +- Paths to be protected. + +![protected-paths](../../user/admin_area/settings/img/protected_paths.png) + +Requests over the rate limit are logged into `auth.log`. diff --git a/doc/administration/settings/push_event_activities_limit.md b/doc/administration/settings/push_event_activities_limit.md new file mode 100644 index 00000000000..7d3a66fb0c4 --- /dev/null +++ b/doc/administration/settings/push_event_activities_limit.md @@ -0,0 +1,38 @@ +--- +stage: Create +group: Source Code +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" +type: reference +--- + +# Push event activities limit and bulk push events **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31007) in GitLab 12.4. + +Set the number of branches or tags to limit the number of single push events +allowed at once. If the number of events is greater than this, GitLab creates +bulk push event instead. + +For example, if 4 branches are pushed and the limit is currently set to 3, +the activity feed displays: + +![Bulk push event](../../user/admin_area/settings/img/bulk_push_event_v12_4.png) + +With this feature, when a single push includes a lot of changes (for example, 1,000 +branches), only 1 bulk push event is created instead of 1,000 push +events. This helps in maintaining good system performance and preventing spam on +the activity feed. + +To modify this setting: + +- In the Admin Area: + 1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). + 1. Select **Admin Area**. + 1. Select **Settings > Network**. + 1. Expand **Performance optimization**. +- Through the [Application settings API](../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls) + as `push_event_activities_limit`. + +The default value is `3`, but the value can be greater than or equal to `0`. Setting this value to `0` does not disable throttling. + +![Push event activities limit](../../user/admin_area/settings/img/push_event_activities_limit_v12_4.png) diff --git a/doc/administration/settings/rate_limit_on_issues_creation.md b/doc/administration/settings/rate_limit_on_issues_creation.md new file mode 100644 index 00000000000..1e19bbea969 --- /dev/null +++ b/doc/administration/settings/rate_limit_on_issues_creation.md @@ -0,0 +1,36 @@ +--- +type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Rate limits on issue creation **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28129) in GitLab 12.10. + +This setting allows you to rate limit the requests to the issue and epic creation endpoints. +To can change its value: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. Select **Settings > Network**. +1. Expand **Issues Rate Limits**. +1. Under **Max requests per minute**, enter the new value. +1. Select **Save changes**. + +For example, if you set a limit of 300, requests using the +[Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/projects/issues_controller.rb) +action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. + +When using [epics](../../user/group/epics/index.md), epic creation shares this rate limit with issues. + +![Rate limits on issues creation](../../user/admin_area/settings/img/rate_limit_on_issues_creation_v14_2.png) + +This limit is: + +- Applied independently per project and per user. +- Not applied per IP address. +- Disabled by default. To enable it, set the option to any value other than `0`. + +Requests over the rate limit are logged into the `auth.log` file. diff --git a/doc/administration/settings/rate_limit_on_notes_creation.md b/doc/administration/settings/rate_limit_on_notes_creation.md new file mode 100644 index 00000000000..59548836e78 --- /dev/null +++ b/doc/administration/settings/rate_limit_on_notes_creation.md @@ -0,0 +1,35 @@ +--- +type: reference +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Rate limits on note creation **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53637) in GitLab 13.9. + +You can configure the per-user rate limit for requests to the note creation endpoint. + +To change the note creation rate limit: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. Select **Settings > Network**. +1. Expand **Notes rate limit**. +1. In the **Maximum requests per minute** box, enter the new value. +1. Optional. In the **Users to exclude from the rate limit** box, list users allowed to exceed the limit. +1. Select **Save changes**. + +This limit is: + +- Applied independently per user. +- Not applied per IP address. + +The default value is `300`. + +Requests over the rate limit are logged into the `auth.log` file. + +For example, if you set a limit of 300, requests using the +[Projects::NotesController#create](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/notes_controller.rb) +action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. diff --git a/doc/administration/settings/rate_limit_on_pipelines_creation.md b/doc/administration/settings/rate_limit_on_pipelines_creation.md new file mode 100644 index 00000000000..19e1410ef73 --- /dev/null +++ b/doc/administration/settings/rate_limit_on_pipelines_creation.md @@ -0,0 +1,35 @@ +--- +type: reference +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Rate limits on pipeline creation **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362475) in GitLab 15.0. + +You can set a limit so that users and processes can't request more than a certain number of pipelines each minute. This limit can help save resources and improve stability. + +For example, if you set a limit of `10`, and `11` requests are sent to the [trigger API](../../ci/triggers/index.md) within one minute, +the eleventh request is blocked. Access to the endpoint is allowed again after one minute. + +This limit is: + +- Applied to the number of pipelines created for the same combination of project, commit, and user. +- Not applied per IP address. +- Disabled by default. + +Requests that exceed the limit are logged in the `application_json.log` file. + +## Set a pipeline request limit + +To limit the number of pipeline requests: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. Select **Settings > Network**. +1. Expand **Pipelines Rate Limits**. +1. Under **Max requests per minute**, enter a value greater than `0`. +1. Select **Save changes**. +1. Enable `ci_enforce_throttle_pipelines_creation` feature flag to enable the rate limit. diff --git a/doc/api/project_relations_export.md b/doc/api/project_relations_export.md index d0839cae40a..e209259c6cc 100644 --- a/doc/api/project_relations_export.md +++ b/doc/api/project_relations_export.md @@ -6,8 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Project relations export API **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70330) in GitLab 14.4 [with a flag](../administration/feature_flags.md) named `bulk_import`. Disabled by default. -> - New application setting `bulk_import_enabled` introduced in GitLab 15.8. Feature flag `bulk_import` removed. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70330) in GitLab 14.4 behind + the `bulk_import` [feature flag](../administration/feature_flags.md), disabled by default. +> - New application setting `bulk_import_enabled` introduced in GitLab 15.8. `bulk_import` feature + flag removed. The project relations export API partially exports a project's structure as separate files for each top-level diff --git a/doc/ci/cloud_services/azure/index.md b/doc/ci/cloud_services/azure/index.md index 03aabadc850..b921dabc4e2 100644 --- a/doc/ci/cloud_services/azure/index.md +++ b/doc/ci/cloud_services/azure/index.md @@ -123,8 +123,11 @@ variables: AZURE_TENANT_ID: "<tenant-id>" auth: + id_tokens: + GITLAB_OIDC_TOKEN: + aud: https://gitlab.com script: - - az login --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID --federated-token $CI_JOB_JWT_V2 + - az login --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID --federated-token $GITLAB_OIDC_TOKEN - az account show ``` @@ -133,7 +136,7 @@ The CI/CD variables are: - `AZURE_CLIENT_ID`: The [application client ID you saved earlier](#create-azure-ad-application-and-service-principal). - `AZURE_TENANT_ID`: Your Azure Active Directory. You can [find it by using the Azure CLI or Azure Portal](https://learn.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant). -- `CI_JOB_JWT_V2`: The JSON web token is a [predefined CI/CD variable](../../variables/predefined_variables.md). +- `GITLAB_OIDC_TOKEN`: An OIDC [ID token](../../yaml/index.md#id_tokens). ## Troubleshooting diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index 8be46da3fa8..e15e09b27c1 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -170,23 +170,3 @@ For more information, see [Custom CI/CD configuration path](../pipelines/setting ## Require an approval before deploying Before promoting a deployment to a production environment, cross-verifying it with a dedicated testing group is an effective way to ensure safety. For more information, see [Deployment Approvals](deployment_approvals.md). - -## Troubleshooting - -### Pipelines jobs fail with `The deployment job is older than the previously succeeded deployment job...` - -This is caused by the [Prevent outdated deployment jobs](../pipelines/settings.md#prevent-outdated-deployment-jobs) feature. -If you have multiple jobs for the same environment (including non-deployment jobs), you might encounter this problem, for example: - -```yaml -build:service-a: - environment: - name: production - -build:service-b: - environment: - name: production -``` - -The [Prevent outdated deployment jobs](../pipelines/settings.md#prevent-outdated-deployment-jobs) might -not work well with this configuration, and must be disabled. diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 23717bb352a..bd4ded8ca11 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -381,7 +381,7 @@ If you use an acronym, spell it out on first use on a page. You do not need to s ### Numbers -When using numbers in text, spell out zero through nine, and use numbers for 10 and greater. For details, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/numbers). +For numbers in text, spell out zero through nine and use numbers for 10 and greater. For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/numbers). ## Text diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index 5e662051701..84ab2ecc4e9 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -152,7 +152,7 @@ Instead of: - This feature enables users to add files to their repository. This phrasing is more active and is from the user perspective, rather than the person who implemented the feature. -[View details in the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/allow-allows). +For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/allow-allows). ## analytics @@ -169,8 +169,8 @@ Instead of **and/or**, use **or** or rewrite the sentence to spell out both opti ## and so on -Do not use **and so on**. Instead, be more specific. For details, see -[the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/and-so-on). +Do not use **and so on**. Instead, be more specific. For more information, see the +[Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/and-so-on). ## area @@ -336,11 +336,11 @@ For more information, see [epic 2150](https://gitlab.com/groups/gitlab-com/-/epi ## confirmation dialog -Use **confirmation dialog** to describe the dialog that asks you to confirm your action. For example: +Use **confirmation dialog** to describe the dialog that asks you to confirm an action. For example: - On the confirmation dialog, select **OK**. -Do not use **confirmation box** or **confirmation dialog box**. +Do not use **confirmation box** or **confirmation dialog box**. See also [**dialog**](#dialog). ## Container Registry @@ -401,9 +401,29 @@ When writing about the Developer role: Do not use **Developer permissions**. A user who is assigned the Developer role has a set of associated permissions. +## dialog + +Use **dialog** rather than any of these alternatives: + +- **dialog box** +- **modal** +- **modal dialog** +- **modal window** +- **pop-up** +- **pop-up window** +- **window** + +See also [**confirmation dialog**](#confirmation-dialog). For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/d/dialog-box-dialog-dialogue). + +When the dialog is the location of an action, use **on** as a preposition. For example: + +- On the **Grant permission** dialog, select **Group**. + +See also [**on**](#on). + ## disable -See [the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/d/disable-disabled) for guidance on **disable**. +See the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/d/disable-disabled) for guidance on **disable**. Use **inactive** or **off** instead. ([Vale](../testing.md#vale) rule: [`InclusionAbleism.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionAbleism.yml)) ## disallow @@ -456,7 +476,7 @@ Do not use Latin abbreviations. Use **for example**, **such as**, **for instance ## ellipsis When documenting UI text, if the UI includes an ellipsis, do not include the ellipsis in the documentation. -For details, see [the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/punctuation/ellipses). +For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/punctuation/ellipses). Use: @@ -476,7 +496,7 @@ Use **emoji** to refer to the plural form of **emoji**. ## enable -See [the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/e/enable-enables) for guidance on **enable**. +See the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/e/enable-enables) for guidance on **enable**. Use **active** or **on** instead. ([Vale](../testing.md#vale) rule: [`InclusionAbleism.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/InclusionAbleism.yml)) ## enter @@ -1048,15 +1068,12 @@ For more information, see the ## on -When documenting how to select high-level UI elements, use the word **on**. - -Use: - -- `On the left sidebar...` +When documenting high-level UI elements, use **on** as a preposition. For example: -Instead of: +- On the left sidebar, select **Settings > CI/CD**. +- On the **Grant permission** dialog, select **Group**. -- Do not: `From the left sidebar...` or `In the left sidebar...` +Do not use **from** or **in**. For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/f/from-vs-on). ## once @@ -1137,7 +1154,7 @@ Use lowercase for **personal access token**. ## please -Do not use **please**. For details, see the [Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). +Do not use **please**. For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). ## Premium @@ -1514,8 +1531,8 @@ in the context of other subscription tiers, follow [the subscription tier](#subs Use a space between the number and the unit of measurement. For example, **128 GB**. ([Vale](../testing.md#vale) rule: [`Units.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Units.yml)) -For other guidance, follow -[the Microsoft style guidelines](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/term-collections/bits-bytes-terms). +For more information, see the +[Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/term-collections/bits-bytes-terms). ## update @@ -1551,7 +1568,7 @@ If the UI element is not in a corner, use **upper left** and **upper right**. Do not use **top left** and **top right**. -For details, see the [Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/u/upper-left-upper-right). +For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/u/upper-left-upper-right). ## useful @@ -1598,7 +1615,7 @@ Instead of: - While job 1 can run quickly, job 2 is more precise. -For details, see the [Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/w/while). +For more information, see the [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/w/while). ## whilst diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index 58f54c399df..de079d08d3a 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -1,43 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference, concepts +redirect_to: '../../administration/merge_requests_approvals.md' +remove_date: '2023-10-12' --- -# Merge request approvals **(PREMIUM SELF)** +This document was moved to [another location](../../administration/merge_requests_approvals.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39060) in GitLab 12.8. - -Merge request approval rules prevent users from overriding certain settings on the project level. -When enabled at the instance level, these settings [cascade](../project/merge_requests/approvals/settings.md#settings-cascading) -and can no longer be changed: - -- In projects. -- In groups. Cascading to groups was [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) - in GitLab 14.5. - -To enable merge request approval settings for an instance: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. Select **Push Rules**. -1. Expand **Merge request approvals**. -1. Choose the required options. -1. Select **Save changes**. - -## Available rules - -Merge request approval settings that can be set at an instance level are: - -- **Prevent approval by author**. Prevents project maintainers from allowing request authors to - merge their own merge requests. -- **Prevent approvals by users who add commits**. Prevents project maintainers from allowing users - to approve merge requests if they have submitted any commits to the source branch. -- **Prevent editing approval rules in projects and merge requests**. Prevents users from modifying - the approvers list in project settings or in individual merge requests. - -See also the following, which are affected by instance-level rules: - -- [Project merge request approval rules](../project/merge_requests/approvals/index.md). -- [Group merge request approval settings](../group/manage.md#group-merge-request-approval-settings) available in GitLab 13.9 and later. +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/settings/protected_paths.md b/doc/user/admin_area/settings/protected_paths.md index 4080ee6a540..519d035244a 100644 --- a/doc/user/admin_area/settings/protected_paths.md +++ b/doc/user/admin_area/settings/protected_paths.md @@ -1,43 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference +redirect_to: '../../../administration/settings/protected_paths.md' +remove_date: '2023-10-12' --- -# Protected paths **(FREE SELF)** +This document was moved to [another location](../../../administration/settings/protected_paths.md). -Rate limiting is a technique that improves the security and durability of a web -application. For more details, see [Rate limits](../../../security/rate_limits.md). - -You can rate limit (protect) specified paths. For these paths, GitLab responds with HTTP status -code `429` to POST requests at protected paths that exceed 10 requests per minute per IP address. - -For example, the following are limited to a maximum 10 requests per minute: - -- User sign-in -- User sign-up (if enabled) -- User password reset - -After 10 requests, the client must wait 60 seconds before it can try again. - -See also: - -- List of paths [protected by default](../../../administration/instance_limits.md#by-protected-path). -- [User and IP rate limits](../../admin_area/settings/user_and_ip_rate_limits.md#response-headers) - for the headers returned to blocked requests. - -## Configure protected paths - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31246) in GitLab 12.4. - -Throttling of protected paths is enabled by default and can be disabled or -customized on **Admin > Network > Protected Paths**, along with these options: - -- Maximum number of requests per period per user. -- Rate limit period in seconds. -- Paths to be protected. - -![protected-paths](img/protected_paths.png) - -Requests over the rate limit are logged into `auth.log`. +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/settings/push_event_activities_limit.md b/doc/user/admin_area/settings/push_event_activities_limit.md index 56a2902f2d9..b7e059cf820 100644 --- a/doc/user/admin_area/settings/push_event_activities_limit.md +++ b/doc/user/admin_area/settings/push_event_activities_limit.md @@ -1,38 +1,11 @@ --- -stage: Create -group: Source Code -info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" -type: reference +redirect_to: '../../../administration/settings/push_event_activities_limit.md' +remove_date: '2023-10-12' --- -# Push event activities limit and bulk push events **(FREE)** +This document was moved to [another location](../../../administration/settings/push_event_activities_limit.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31007) in GitLab 12.4. - -Set the number of branches or tags to limit the number of single push events -allowed at once. If the number of events is greater than this, GitLab creates -bulk push event instead. - -For example, if 4 branches are pushed and the limit is currently set to 3, -the activity feed displays: - -![Bulk push event](img/bulk_push_event_v12_4.png) - -With this feature, when a single push includes a lot of changes (for example, 1,000 -branches), only 1 bulk push event is created instead of 1,000 push -events. This helps in maintaining good system performance and preventing spam on -the activity feed. - -To modify this setting: - -- In the Admin Area: - 1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). - 1. Select **Admin Area**. - 1. Select **Settings > Network**. - 1. Expand **Performance optimization**. -- Through the [Application settings API](../../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls) - as `push_event_activities_limit`. - -The default value is `3`, but the value can be greater than or equal to `0`. Setting this value to `0` does not disable throttling. - -![Push event activities limit](img/push_event_activities_limit_v12_4.png) +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md index 09a1e023c2b..aca30177c54 100644 --- a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md @@ -1,36 +1,11 @@ --- -type: reference -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../../administration/settings/rate_limit_on_issues_creation.md' +remove_date: '2023-10-12' --- -# Rate limits on issue creation **(FREE SELF)** +This document was moved to [another location](../../../administration/settings/rate_limit_on_issues_creation.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28129) in GitLab 12.10. - -This setting allows you to rate limit the requests to the issue and epic creation endpoints. -To can change its value: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. Select **Settings > Network**. -1. Expand **Issues Rate Limits**. -1. Under **Max requests per minute**, enter the new value. -1. Select **Save changes**. - -For example, if you set a limit of 300, requests using the -[Projects::IssuesController#create](https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/projects/issues_controller.rb) -action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. - -When using [epics](../../group/epics/index.md), epic creation shares this rate limit with issues. - -![Rate limits on issues creation](img/rate_limit_on_issues_creation_v14_2.png) - -This limit is: - -- Applied independently per project and per user. -- Not applied per IP address. -- Disabled by default. To enable it, set the option to any value other than `0`. - -Requests over the rate limit are logged into the `auth.log` file. +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/settings/rate_limit_on_notes_creation.md b/doc/user/admin_area/settings/rate_limit_on_notes_creation.md index 59548836e78..6d5c93f8554 100644 --- a/doc/user/admin_area/settings/rate_limit_on_notes_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_notes_creation.md @@ -1,35 +1,11 @@ --- -type: reference -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../../administration/settings/rate_limit_on_notes_creation.md' +remove_date: '2023-10-12' --- -# Rate limits on note creation **(FREE SELF)** +This document was moved to [another location](../../../administration/settings/rate_limit_on_notes_creation.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53637) in GitLab 13.9. - -You can configure the per-user rate limit for requests to the note creation endpoint. - -To change the note creation rate limit: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. Select **Settings > Network**. -1. Expand **Notes rate limit**. -1. In the **Maximum requests per minute** box, enter the new value. -1. Optional. In the **Users to exclude from the rate limit** box, list users allowed to exceed the limit. -1. Select **Save changes**. - -This limit is: - -- Applied independently per user. -- Not applied per IP address. - -The default value is `300`. - -Requests over the rate limit are logged into the `auth.log` file. - -For example, if you set a limit of 300, requests using the -[Projects::NotesController#create](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/notes_controller.rb) -action exceeding a rate of 300 per minute are blocked. Access to the endpoint is allowed after one minute. +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md index 2d0c4405211..c469a77f7d2 100644 --- a/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md @@ -1,35 +1,11 @@ --- -type: reference -stage: Verify -group: Pipeline Execution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../../administration/settings/rate_limit_on_pipelines_creation.md' +remove_date: '2023-10-12' --- -# Rate limits on pipeline creation **(FREE SELF)** +This document was moved to [another location](../../../administration/settings/rate_limit_on_pipelines_creation.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362475) in GitLab 15.0. - -You can set a limit so that users and processes can't request more than a certain number of pipelines each minute. This limit can help save resources and improve stability. - -For example, if you set a limit of `10`, and `11` requests are sent to the [trigger API](../../../ci/triggers/index.md) within one minute, -the eleventh request is blocked. Access to the endpoint is allowed again after one minute. - -This limit is: - -- Applied to the number of pipelines created for the same combination of project, commit, and user. -- Not applied per IP address. -- Disabled by default. - -Requests that exceed the limit are logged in the `application_json.log` file. - -## Set a pipeline request limit - -To limit the number of pipeline requests: - -1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). -1. Select **Admin Area**. -1. Select **Settings > Network**. -1. Expand **Pipelines Rate Limits**. -1. Under **Max requests per minute**, enter a value greater than `0`. -1. Select **Save changes**. -1. Enable `ci_enforce_throttle_pipelines_creation` feature flag to enable the rate limit. +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/clusters/agent/gitops/flux_tutorial.md b/doc/user/clusters/agent/gitops/flux_tutorial.md index b9bb4a410d6..8aee0c01d65 100644 --- a/doc/user/clusters/agent/gitops/flux_tutorial.md +++ b/doc/user/clusters/agent/gitops/flux_tutorial.md @@ -180,8 +180,6 @@ To demonstrate, deploy an `nginx` application and point Flux at it: interval: 1m0s ref: branch: main - secretRef: - name: example-nginx-app url: https://gitlab.com/gitlab-examples/ops/gitops-demo/example-mini-flux-deployment.git --- apiVersion: kustomize.toolkit.fluxcd.io/v1 diff --git a/doc/user/project/integrations/mlflow_client.md b/doc/user/project/integrations/mlflow_client.md new file mode 100644 index 00000000000..ce092d10d20 --- /dev/null +++ b/doc/user/project/integrations/mlflow_client.md @@ -0,0 +1,11 @@ +--- +redirect_to: '../ml/experiment_tracking/mlflow_client.md' +remove_date: '2023-10-12' +--- + +This document was moved to [another location](../ml/experiment_tracking/mlflow_client.md). + +<!-- This redirect file can be deleted after <2023-10-12>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index a7c8f1b2d8c..d6dc707dbf8 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -24,6 +24,7 @@ Event type | Trigger [Subgroup event](#subgroup-events) | A subgroup is created or removed from a group. [Feature flag event](#feature-flag-events) | A feature flag is turned on or off. [Release event](#release-events) | A release is created or updated. +[Emoji event](#emoji-events) | An emoji is awarded or revoked. NOTE: If an author has no public email listed in their @@ -1847,3 +1848,149 @@ Payload example: } } ``` + +## Emoji events + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123952) in GitLab 16.2 [with a flag](../../../administration/feature_flags.md) named `emoji_webhooks`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `emoji_webhooks`. +On GitLab.com, this feature is not available. +This feature is not ready for production use. + +NOTE: +To have the `emoji_webhooks` flag enabled on GitLab.com, see [issue 417288](https://gitlab.com/gitlab-org/gitlab/-/issues/417288). + +An emoji event is triggered when an emoji is awarded or revoked on: + +- Issues +- Merge requests +- Project snippets +- Comments on: + - Issues + - Merge requests + - Project snippets + - Commits + +The available values for `object_attributes.action` in the payload are: + +- `award` +- `revoke` + +Request header: + +```plaintext +X-Gitlab-Event: Emoji Hook +``` + +Payload example: + +```json +{ + "object_kind": "emoji", + "event_type": "award", + "user": { + "id": 1, + "name": "Blake Bergstrom", + "username": "root", + "avatar_url": "http://example.com/uploads/-/system/user/avatar/1/avatar.png", + "email": "[REDACTED]" + }, + "project_id": 6, + "project": { + "id": 6, + "name": "Flight", + "description": "Velit fugit aperiam illum deleniti odio sequi.", + "web_url": "http://example.com/flightjs/Flight", + "avatar_url": null, + "git_ssh_url": "ssh://git@example.com/flightjs/Flight.git", + "git_http_url": "http://example.com/flightjs/Flight.git", + "namespace": "Flightjs", + "visibility_level": 20, + "path_with_namespace": "flightjs/Flight", + "default_branch": "master", + "ci_config_path": null, + "homepage": "http://example.com/flightjs/Flight", + "url": "ssh://git@example.com/flightjs/Flight.git", + "ssh_url": "ssh://git@example.com/flightjs/Flight.git", + "http_url": "http://example.com/flightjs/Flight.git" + }, + "object_attributes": { + "user_id": 1, + "created_at": "2023-07-04 20:44:11 UTC", + "id": 1, + "name": "thumbsup", + "awardable_type": "Note", + "awardable_id": 363, + "updated_at": "2023-07-04 20:44:11 UTC", + "action": "award", + "awarded_on_url": "http://example.com/flightjs/Flight/-/issues/42#note_363" + }, + "note": { + "attachment": null, + "author_id": 1, + "change_position": null, + "commit_id": null, + "created_at": "2023-07-04 15:09:55 UTC", + "discussion_id": "c3d97fd471f210a5dc8b97a409e3bea95ee06c14", + "id": 363, + "line_code": null, + "note": "Testing 123", + "noteable_id": 635, + "noteable_type": "Issue", + "original_position": null, + "position": null, + "project_id": 6, + "resolved_at": null, + "resolved_by_id": null, + "resolved_by_push": null, + "st_diff": null, + "system": false, + "type": null, + "updated_at": "2023-07-04 19:58:46 UTC", + "updated_by_id": null, + "description": "Testing 123", + "url": "http://example.com/flightjs/Flight/-/issues/42#note_363" + }, + "issue": { + "author_id": 1, + "closed_at": null, + "confidential": false, + "created_at": "2023-07-04 14:59:43 UTC", + "description": "Issue description!", + "discussion_locked": null, + "due_date": null, + "id": 635, + "iid": 42, + "last_edited_at": null, + "last_edited_by_id": null, + "milestone_id": null, + "moved_to_id": null, + "duplicated_to_id": null, + "project_id": 6, + "relative_position": 18981, + "state_id": 1, + "time_estimate": 0, + "title": "New issue!", + "updated_at": "2023-07-04 15:09:55 UTC", + "updated_by_id": null, + "weight": null, + "health_status": null, + "url": "http://example.com/flightjs/Flight/-/issues/42", + "total_time_spent": 0, + "time_change": 0, + "human_total_time_spent": null, + "human_time_change": null, + "human_time_estimate": null, + "assignee_ids": [ + 1 + ], + "assignee_id": 1, + "labels": [ + + ], + "state": "opened", + "severity": "unknown" + } +} +``` diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 418c43b343f..e7df93dbc1c 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -43,8 +43,8 @@ Meanwhile: ## Configure Service Desk -To start using Service Desk for a project, you must first turn it on. -By default, Service Desk is turned off. +By default, Service Desk is active in new projects. +If it's not active, you can do it in the project's settings. Prerequisites: diff --git a/gems/gem.gitlab-ci.yml b/gems/gem.gitlab-ci.yml index 8a0518fa415..10905d8c243 100644 --- a/gems/gem.gitlab-ci.yml +++ b/gems/gem.gitlab-ci.yml @@ -1,4 +1,5 @@ -# The template generates jobs for gems vendored in the main GitLab project under `gems/`. +# The template generates jobs for gems vendored in the main GitLab project +# under `gem_path_prefix` (defaults to `gems/`). # # Inputs # - `gem_name`: The name of the gem, i.e. if the gem is located at `gems/gitlab-rspec`, `gem_name` should be set to `gitlab-rspec`. @@ -17,13 +18,15 @@ workflow: PIPELINE_NAME: '[$[[inputs.gem_name]] gem] Ruby $RUBY_VERSION $CI_MERGE_REQUEST_EVENT_TYPE MR pipeline' variables: + BUNDLE_PATH: "vendor" + BUNDLE_FROZEN: "true" GIT_DEPTH: "20" # 'GIT_STRATEGY: clone' optimizes the pack-objects cache hit ratio GIT_STRATEGY: "clone" GIT_SUBMODULE_STRATEGY: "none" GET_SOURCES_ATTEMPTS: "3" # Default Ruby version for jobs that don't use .ruby_matrix - RUBY_VERSION: "3.1" + RUBY_VERSION: "3.0" default: image: "ruby:${RUBY_VERSION}" @@ -35,9 +38,7 @@ default: - cd $[[inputs.gem_path_prefix]]$[[inputs.gem_name]] - ruby -v # Print out ruby version for debugging - bundle_version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | sed -e 's/[[:space:]]//') - - gem install bundler --version $bundle_version --no-document # Bundler is not installed with the image - - bundle config set --local path 'vendor' # Install dependencies into ./vendor/ruby - - bundle config set --local frozen 'true' # Disallow Gemfile.lock changes on CI + - gem install bundler --version "$bundle_version" --no-document # Bundler is not installed with the image - bundle config # Show bundler configuration - bundle install --jobs=$(nproc) --retry=3 diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index f7a39db7249..aa7468723b7 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -82,7 +82,9 @@ module API unauthorized! unless award.user == current_user || current_user&.can_admin_all_resources? - destroy_conditionally!(award) + destroy_conditionally!(award) do + AwardEmojis::DestroyService.new(awardable, award.name, award.user).execute + end end end end diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb index bffb057abed..b85d2747226 100644 --- a/lib/api/entities/project_hook.rb +++ b/lib/api/entities/project_hook.rb @@ -14,6 +14,7 @@ module API expose :job_events, documentation: { type: 'boolean' } expose :releases_events, documentation: { type: 'boolean' } expose :push_events_branch_filter, documentation: { type: 'string', example: 'my-branch-*' } + expose :emoji_events, documentation: { type: 'boolean' } end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index db97f4988e1..c9cba397f5c 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -31,6 +31,7 @@ module API optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :deployment_events, type: Boolean, desc: "Trigger hook on deployment events" optional :releases_events, type: Boolean, desc: "Trigger hook on release events" + optional :emoji_events, type: Boolean, desc: "Trigger hook on emoji events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index 04febebff8e..e1a0e5cf2fb 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -33,7 +33,7 @@ module BulkImports def download_service(context) @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: filename ) diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb index 68bd64dc2ff..0bf4d341aad 100644 --- a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb @@ -6,6 +6,8 @@ module BulkImports class LfsObjectsPipeline include Pipeline + file_extraction_pipeline! + def extract(_context) download_service.execute decompression_service.execute @@ -48,7 +50,7 @@ module BulkImports def download_service BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: targz_filename ) diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index 06132791ea6..81ce20db9ab 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -10,6 +10,8 @@ module BulkImports AvatarLoadingError = Class.new(StandardError) + file_extraction_pipeline! + def extract(_context) download_service.execute decompression_service.execute @@ -46,7 +48,7 @@ module BulkImports def download_service BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(relation), + relative_url: context.entity.relation_download_url_path(relation, context.extra[:batch_number]), tmpdir: tmpdir, filename: targz_filename ) diff --git a/lib/gitlab/data_builder/emoji.rb b/lib/gitlab/data_builder/emoji.rb new file mode 100644 index 00000000000..63562eca155 --- /dev/null +++ b/lib/gitlab/data_builder/emoji.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Emoji + extend self + + def build(award_emoji, user, action) + project = award_emoji.awardable.project + data = build_base_data(project, user, award_emoji, action) + + if award_emoji.awardable.is_a?(::Note) + note = award_emoji.awardable + data[:note] = note.hook_attrs + noteable = note.noteable + else + noteable = award_emoji.awardable + end + + if noteable.respond_to?(:hook_attrs) + data[noteable.class.underscore.to_sym] = noteable.hook_attrs + else + Gitlab::AppLogger.error( + "Error building payload data for emoji webhook. #{noteable.class} does not respond to hook_attrs.") + end + + data + end + + def build_base_data(project, user, award_emoji, action) + base_data = { + object_kind: 'emoji', + event_type: action, + user: user.hook_attrs, + project_id: project.id, + project: project.hook_attrs, + object_attributes: award_emoji.hook_attrs + } + + base_data[:object_attributes][:awarded_on_url] = Gitlab::UrlBuilder.build(award_emoji.awardable) + base_data + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index d7128a20a0b..db05635c73d 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -95,7 +95,7 @@ module Gitlab async_index = Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec| rec.table_name = table_name - rec.definition = definition + rec.definition = definition.to_s.strip end Gitlab::AppLogger.info( diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index a3c600a4519..98eb282e43f 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -13,6 +13,8 @@ module Gitlab MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH MAX_DEFINITION_LENGTH = 2048 + before_validation :remove_whitespaces + validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } validates :table_name, presence: true, length: { maximum: MAX_TABLE_NAME_LENGTH } validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } @@ -29,6 +31,10 @@ module Gitlab private + def remove_whitespaces + definition.strip! if definition.present? + end + def ensure_correct_schema_and_table_name return unless table_name diff --git a/lib/gitlab/hook_data/emoji_builder.rb b/lib/gitlab/hook_data/emoji_builder.rb new file mode 100644 index 00000000000..673eb516e43 --- /dev/null +++ b/lib/gitlab/hook_data/emoji_builder.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class EmojiBuilder < BaseBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + user_id + created_at + id + name + awardable_type + awardable_id + updated_at + ].freeze + + alias_method :award_emoji, :object + + def build + award_emoji + .attributes + .with_indifferent_access + .slice(*SAFE_HOOK_ATTRIBUTES) + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2f0b9362ab9..e43683b4b51 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -12644,6 +12644,9 @@ msgstr "" msgid "ContributionEvent|Left project %{resourceParentLink}." msgstr "" +msgid "ContributionEvent|Made a private contribution." +msgstr "" + msgid "ContributionEvent|Pushed a new branch %{refLink} in %{resourceParentLink}." msgstr "" @@ -51324,6 +51327,9 @@ msgstr "" msgid "Webhooks|A wiki page is created or updated." msgstr "" +msgid "Webhooks|An emoji is awarded or revoked. %{help_link}?" +msgstr "" + msgid "Webhooks|An issue is created, updated, closed, or reopened." msgstr "" @@ -51606,6 +51612,9 @@ msgstr "" msgid "Which API requests are affected?" msgstr "" +msgid "Which emoji events trigger webhooks" +msgstr "" + msgid "While it's rare to have no vulnerabilities, it can happen. In any event, we ask that you please double check your settings to make sure you've set up your dashboard correctly." msgstr "" diff --git a/spec/factories/bulk_import/trackers.rb b/spec/factories/bulk_import/trackers.rb index 3e69ab26801..3d5d88954ed 100644 --- a/spec/factories/bulk_import/trackers.rb +++ b/spec/factories/bulk_import/trackers.rb @@ -24,5 +24,13 @@ FactoryBot.define do trait :skipped do status { -2 } end + + trait :batched do + batched { true } + end + + trait :stale do + created_at { 1.day.ago } + end end end diff --git a/spec/factories/project_hooks.rb b/spec/factories/project_hooks.rb index 3e70b897df6..34797bd933e 100644 --- a/spec/factories/project_hooks.rb +++ b/spec/factories/project_hooks.rb @@ -28,6 +28,7 @@ FactoryBot.define do deployment_events { true } feature_flag_events { true } releases_events { true } + emoji_events { true } end trait :with_push_branch_filter do diff --git a/spec/features/projects/settings/webhooks_settings_spec.rb b/spec/features/projects/settings/webhooks_settings_spec.rb index 5d345c63d60..e3d09ba4ded 100644 --- a/spec/features/projects/settings/webhooks_settings_spec.rb +++ b/spec/features/projects/settings/webhooks_settings_spec.rb @@ -46,6 +46,7 @@ RSpec.describe 'Projects > Settings > Webhook Settings', feature_category: :grou expect(page).to have_content('Pipeline events') expect(page).to have_content('Wiki page events') expect(page).to have_content('Releases events') + expect(page).to have_content('Emoji events') end it 'create webhook', :js do diff --git a/spec/fixtures/api/schemas/public_api/v4/project_hook.json b/spec/fixtures/api/schemas/public_api/v4/project_hook.json index 6070f3a55f9..b89f5af8078 100644 --- a/spec/fixtures/api/schemas/public_api/v4/project_hook.json +++ b/spec/fixtures/api/schemas/public_api/v4/project_hook.json @@ -22,38 +22,106 @@ "releases_events", "alert_status", "disabled_until", - "url_variables" + "url_variables", + "emoji_events" ], "properties": { - "id": { "type": "integer" }, - "project_id": { "type": "integer" }, - "url": { "type": "string" }, - "created_at": { "type": "string", "format": "date-time" }, - "push_events": { "type": "boolean" }, - "push_events_branch_filter": { "type": ["string", "null"] }, - "tag_push_events": { "type": "boolean" }, - "merge_requests_events": { "type": "boolean" }, - "repository_update_events": { "type": "boolean" }, - "enable_ssl_verification": { "type": "boolean" }, - "issues_events": { "type": "boolean" }, - "confidential_issues_events": { "type": ["boolean", "null"] }, - "note_events": { "type": "boolean" }, - "confidential_note_events": { "type": ["boolean", "null"] }, - "pipeline_events": { "type": "boolean" }, - "wiki_page_events": { "type": "boolean" }, - "job_events": { "type": "boolean" }, - "deployment_events": { "type": "boolean" }, - "releases_events": { "type": "boolean" }, - "alert_status": { "type": "string", "enum": ["executable","disabled","temporarily_disabled"] }, - "disabled_until": { "type": ["string", "null"] }, + "id": { + "type": "integer" + }, + "project_id": { + "type": "integer" + }, + "url": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "push_events": { + "type": "boolean" + }, + "push_events_branch_filter": { + "type": [ + "string", + "null" + ] + }, + "tag_push_events": { + "type": "boolean" + }, + "merge_requests_events": { + "type": "boolean" + }, + "repository_update_events": { + "type": "boolean" + }, + "enable_ssl_verification": { + "type": "boolean" + }, + "issues_events": { + "type": "boolean" + }, + "confidential_issues_events": { + "type": [ + "boolean", + "null" + ] + }, + "note_events": { + "type": "boolean" + }, + "confidential_note_events": { + "type": [ + "boolean", + "null" + ] + }, + "pipeline_events": { + "type": "boolean" + }, + "wiki_page_events": { + "type": "boolean" + }, + "job_events": { + "type": "boolean" + }, + "deployment_events": { + "type": "boolean" + }, + "releases_events": { + "type": "boolean" + }, + "emoji_events": { + "type": "boolean" + }, + "alert_status": { + "type": "string", + "enum": [ + "executable", + "disabled", + "temporarily_disabled" + ] + }, + "disabled_until": { + "type": [ + "string", + "null" + ] + }, "url_variables": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["key"], + "required": [ + "key" + ], "properties": { - "key": { "type": "string" } + "key": { + "type": "string" + } } } } diff --git a/spec/frontend/contribution_events/components/contribution_event/contribution_event_private_spec.js b/spec/frontend/contribution_events/components/contribution_event/contribution_event_private_spec.js new file mode 100644 index 00000000000..42855134a09 --- /dev/null +++ b/spec/frontend/contribution_events/components/contribution_event/contribution_event_private_spec.js @@ -0,0 +1,33 @@ +import { mountExtended } from 'helpers/vue_test_utils_helper'; +import ContributionEventPrivate from '~/contribution_events/components/contribution_event/contribution_event_private.vue'; +import ContributionEventBase from '~/contribution_events/components/contribution_event/contribution_event_base.vue'; +import { eventPrivate } from '../../utils'; + +const defaultPropsData = { + event: eventPrivate(), +}; + +describe('ContributionEventPrivate', () => { + let wrapper; + + const createComponent = () => { + wrapper = mountExtended(ContributionEventPrivate, { + propsData: defaultPropsData, + }); + }; + + beforeEach(() => { + createComponent(); + }); + + it('renders `ContributionEventBase`', () => { + expect(wrapper.findComponent(ContributionEventBase).props()).toMatchObject({ + event: defaultPropsData.event, + iconName: 'eye-slash', + }); + }); + + it('renders message', () => { + expect(wrapper.findByTestId('event-body').text()).toBe(ContributionEventPrivate.i18n.message); + }); +}); diff --git a/spec/frontend/contribution_events/components/contribution_events_spec.js b/spec/frontend/contribution_events/components/contribution_events_spec.js index a1d2570f2f6..4f32d2f6360 100644 --- a/spec/frontend/contribution_events/components/contribution_events_spec.js +++ b/spec/frontend/contribution_events/components/contribution_events_spec.js @@ -5,7 +5,15 @@ import ContributionEventExpired from '~/contribution_events/components/contribut import ContributionEventJoined from '~/contribution_events/components/contribution_event/contribution_event_joined.vue'; import ContributionEventLeft from '~/contribution_events/components/contribution_event/contribution_event_left.vue'; import ContributionEventPushed from '~/contribution_events/components/contribution_event/contribution_event_pushed.vue'; -import { eventApproved, eventExpired, eventJoined, eventLeft, eventPushedBranch } from '../utils'; +import ContributionEventPrivate from '~/contribution_events/components/contribution_event/contribution_event_private.vue'; +import { + eventApproved, + eventExpired, + eventJoined, + eventLeft, + eventPushedBranch, + eventPrivate, +} from '../utils'; describe('ContributionEvents', () => { let wrapper; @@ -13,7 +21,14 @@ describe('ContributionEvents', () => { const createComponent = () => { wrapper = shallowMountExtended(ContributionEvents, { propsData: { - events: [eventApproved(), eventExpired(), eventJoined(), eventLeft(), eventPushedBranch()], + events: [ + eventApproved(), + eventExpired(), + eventJoined(), + eventLeft(), + eventPushedBranch(), + eventPrivate(), + ], }, }); }; @@ -25,6 +40,7 @@ describe('ContributionEvents', () => { ${ContributionEventJoined} | ${eventJoined()} ${ContributionEventLeft} | ${eventLeft()} ${ContributionEventPushed} | ${eventPushedBranch()} + ${ContributionEventPrivate} | ${eventPrivate()} `( 'renders `$expectedComponent.name` component and passes expected event', ({ expectedComponent, expectedEvent }) => { diff --git a/spec/frontend/contribution_events/utils.js b/spec/frontend/contribution_events/utils.js index c9ef2ff2c3e..7cadfd71635 100644 --- a/spec/frontend/contribution_events/utils.js +++ b/spec/frontend/contribution_events/utils.js @@ -5,6 +5,7 @@ import { EVENT_TYPE_JOINED, EVENT_TYPE_LEFT, EVENT_TYPE_PUSHED, + EVENT_TYPE_PRIVATE, PUSH_EVENT_REF_TYPE_BRANCH, PUSH_EVENT_REF_TYPE_TAG, } from '~/contribution_events/constants'; @@ -44,3 +45,5 @@ export const eventPushedRemovedTag = findPushEvent({ refType: PUSH_EVENT_REF_TYPE_TAG, }); export const eventBulkPushedBranch = findPushEvent({ commitCount: 5 }); + +export const eventPrivate = () => ({ ...events[0], action: EVENT_TYPE_PRIVATE }); diff --git a/spec/frontend/profile/components/profile_tabs_spec.js b/spec/frontend/profile/components/profile_tabs_spec.js index f3dda2e205f..3474bbf8d0c 100644 --- a/spec/frontend/profile/components/profile_tabs_spec.js +++ b/spec/frontend/profile/components/profile_tabs_spec.js @@ -4,6 +4,7 @@ import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; import { createAlert } from '~/alert'; import { getUserProjects } from '~/rest_api'; import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; +import { VISIBILITY_LEVEL_PUBLIC_STRING } from '~/visibility_level/constants'; import OverviewTab from '~/profile/components/overview_tab.vue'; import ActivityTab from '~/profile/components/activity_tab.vue'; import GroupsTab from '~/profile/components/groups_tab.vue'; @@ -60,18 +61,30 @@ describe('ProfileTabs', () => { }); describe('when personal projects API request is successful', () => { - beforeEach(async () => { + it('passes correct props to `OverviewTab` component', async () => { getUserProjects.mockResolvedValueOnce({ data: projects }); createComponent(); await waitForPromises(); - }); - it('passes correct props to `OverviewTab` component', () => { expect(wrapper.findComponent(OverviewTab).props()).toMatchObject({ personalProjects: convertObjectPropsToCamelCase(projects, { deep: true }), personalProjectsLoading: false, }); }); + + describe('when projects do not have `visibility` key', () => { + it('sets visibility to public', async () => { + const [{ visibility, ...projectWithoutVisibility }] = projects; + + getUserProjects.mockResolvedValueOnce({ data: [projectWithoutVisibility] }); + createComponent(); + await waitForPromises(); + + expect(wrapper.findComponent(OverviewTab).props('personalProjects')[0].visibility).toBe( + VISIBILITY_LEVEL_PUBLIC_STRING, + ); + }); + }); }); describe('when personal projects API request is not successful', () => { diff --git a/spec/frontend/vue_shared/components/projects_list/projects_list_item_spec.js b/spec/frontend/vue_shared/components/projects_list/projects_list_item_spec.js index 3e4d5c558f6..8d8df3095c6 100644 --- a/spec/frontend/vue_shared/components/projects_list/projects_list_item_spec.js +++ b/spec/frontend/vue_shared/components/projects_list/projects_list_item_spec.js @@ -38,6 +38,7 @@ describe('ProjectsListItem', () => { const findProjectTopics = () => wrapper.findByTestId('project-topics'); const findPopover = () => findProjectTopics().findComponent(GlPopover); const findProjectDescription = () => wrapper.findByTestId('project-description'); + const findVisibilityIcon = () => findAvatarLabeled().findComponent(GlIcon); it('renders project avatar', () => { createComponent(); @@ -66,6 +67,19 @@ describe('ProjectsListItem', () => { expect(tooltip.value).toBe(PROJECT_VISIBILITY_TYPE[VISIBILITY_LEVEL_PRIVATE_STRING]); }); + describe('when visibility is not provided', () => { + it('does not render visibility icon', () => { + const { visibility, ...projectWithoutVisibility } = project; + createComponent({ + propsData: { + project: projectWithoutVisibility, + }, + }); + + expect(findVisibilityIcon().exists()).toBe(false); + }); + }); + it('renders access role badge', () => { createComponent(); @@ -113,6 +127,19 @@ describe('ProjectsListItem', () => { expect(wrapper.findComponent(TimeAgoTooltip).props('time')).toBe(project.updatedAt); }); + describe('when updated at is not available', () => { + it('does not render updated at', () => { + const { updatedAt, ...projectWithoutUpdatedAt } = project; + createComponent({ + propsData: { + project: projectWithoutUpdatedAt, + }, + }); + + expect(wrapper.findComponent(TimeAgoTooltip).exists()).toBe(false); + }); + }); + describe('when issues are enabled', () => { it('renders issues count', () => { createComponent(); diff --git a/spec/graphql/types/detployment_tag_type_spec.rb b/spec/graphql/types/deployment_tag_type_spec.rb index 9a7a8db0970..b6741c208fe 100644 --- a/spec/graphql/types/detployment_tag_type_spec.rb +++ b/spec/graphql/types/deployment_tag_type_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe GitlabSchema.types['DeploymentTag'] do +RSpec.describe GitlabSchema.types['DeploymentTag'], feature_category: :continuous_delivery do specify { expect(described_class.graphql_name).to eq('DeploymentTag') } it 'has the expected fields' do diff --git a/spec/lib/gitlab/data_builder/emoji_spec.rb b/spec/lib/gitlab/data_builder/emoji_spec.rb new file mode 100644 index 00000000000..a9cd9824e55 --- /dev/null +++ b/spec/lib/gitlab/data_builder/emoji_spec.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::DataBuilder::Emoji, feature_category: :team_planning do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } + let_it_be(:issue) { create(:issue, project: project) } + let_it_be(:merge_request) { create(:merge_request, source_project: project) } + let_it_be(:snippet) { create(:snippet, project: project) } + let(:action) { 'award' } + let(:data) { described_class.build(award_emoji, user, action) } + let(:award_emoji) { create(:award_emoji, awardable: awardable) } + + shared_examples 'includes standard data' do + specify do + expect(awardable).to receive(:hook_attrs) + expect(data[:object_attributes]).to have_key(:awarded_on_url) + expect(data[:object_kind]).to eq('emoji') + expect(data[:user]).to eq(user.hook_attrs) + end + + include_examples 'project hook data' + end + + describe 'when emoji on issue' do + let(:awardable) { issue } + + it_behaves_like 'includes standard data' + + it 'returns the issue data' do + expect(awardable).to receive(:hook_attrs) + expect(data).to have_key(:issue) + end + end + + describe 'when emoji on merge request' do + let(:awardable) { merge_request } + + it_behaves_like 'includes standard data' + + it 'returns the merge request data' do + expect(awardable).to receive(:hook_attrs) + expect(data).to have_key(:merge_request) + end + end + + describe 'when emoji on snippet' do + let(:awardable) { snippet } + + it_behaves_like 'includes standard data' + + it 'returns the snippet data' do + expect(awardable).to receive(:hook_attrs) + expect(data).to have_key(:snippet) + end + end + + describe 'when emoji on note' do + describe 'when note on issue' do + let(:note) { create(:note, noteable: issue, project: project) } + let(:awardable) { note } + + it_behaves_like 'includes standard data' + + it 'returns the note and issue data' do + expect(note.noteable).to receive(:hook_attrs) + expect(data).to have_key(:note) + expect(data).to have_key(:issue) + end + end + + describe 'when note on merge request' do + let(:note) { create(:note, noteable: merge_request, project: project) } + let(:awardable) { note } + + it_behaves_like 'includes standard data' + + it 'returns the note and merge request data' do + expect(note.noteable).to receive(:hook_attrs) + expect(data).to have_key(:note) + expect(data).to have_key(:merge_request) + end + end + + describe 'when note on snippet' do + let(:note) { create(:note, noteable: snippet, project: project) } + let(:awardable) { note } + + it_behaves_like 'includes standard data' + + it 'returns the note and snippet data' do + expect(note.noteable).to receive(:hook_attrs) + expect(data).to have_key(:note) + expect(data).to have_key(:snippet) + end + end + + describe 'when note on commit' do + let(:note) { create(:note_on_commit, project: project) } + let(:awardable) { note } + + it_behaves_like 'includes standard data' + + it 'returns the note and commit data' do + expect(note.noteable).to receive(:hook_attrs) + expect(data).to have_key(:note) + expect(data).to have_key(:commit) + end + end + end + + describe 'when awardable does not respond to hook_attrs' do + let(:awardable) { issue } + + it_behaves_like 'includes standard data' + + it 'returns the issue data' do + allow(award_emoji.awardable).to receive(:respond_to?).with(:hook_attrs).and_return(false) + expect(Gitlab::AppLogger).to receive(:error).with( + 'Error building payload data for emoji webhook. Issue does not respond to hook_attrs.') + + data + end + end +end diff --git a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb index b2ba1a60fbb..309bbf1e3f0 100644 --- a/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb @@ -223,6 +223,21 @@ RSpec.describe Gitlab::Database::AsyncIndexes::MigrationHelpers, feature_categor expect(async_index).to have_attributes(table_name: table_name, definition: index_definition) end end + + context 'when the given SQL has whitespace' do + let(:index_definition) { " #{super()}" } + let(:async_index) { index_model.find_by(name: index_name) } + + it 'creates the async index record' do + expect { prepare_async_index_from_sql }.to change { index_model.where(name: index_name).count }.by(1) + end + + it 'sets the async index attributes correctly' do + prepare_async_index_from_sql + + expect(async_index).to have_attributes(table_name: table_name, definition: index_definition.strip) + end + end end end end diff --git a/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb b/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb index 9e37124ba28..7e111dbe08f 100644 --- a/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb +++ b/spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb @@ -55,6 +55,21 @@ RSpec.describe Gitlab::Database::AsyncIndexes::PostgresAsyncIndex, type: :model, it_behaves_like 'table_name is invalid' end + + context 'when passing a definition with beginning or trailing whitespace' do + let(:model) { super().tap { |m| m.definition = definition } } + let(:definition) do + <<-SQL + CREATE UNIQUE INDEX CONCURRENTLY foo_index ON bar_field (uuid); + SQL + end + + it "strips the definition field" do + expect(model).to be_valid + model.save! + expect(model.definition).to eq(definition.strip) + end + end end describe 'scopes' do diff --git a/spec/lib/gitlab/hook_data/emoji_builder_spec.rb b/spec/lib/gitlab/hook_data/emoji_builder_spec.rb new file mode 100644 index 00000000000..9c9f74c7da2 --- /dev/null +++ b/spec/lib/gitlab/hook_data/emoji_builder_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HookData::EmojiBuilder, feature_category: :team_planning do + let_it_be(:award_emoji) { create(:award_emoji) } + + let(:builder) { described_class.new(award_emoji) } + + describe '#build' do + let(:data) { builder.build } + + it 'includes safe attributes' do + expect(data.keys).to match_array( + %w[ + user_id + created_at + id + name + awardable_type + awardable_id + updated_at + ] + ) + end + end +end diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb index 73f8447c191..7252457849e 100644 --- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb @@ -64,6 +64,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory, :use_clean_rails_ 'job_events' => false, 'wiki_page_events' => true, 'releases_events' => false, + 'emoji_events' => false, 'token' => token } end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index dd96bf8c712..b6328994c5b 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -608,6 +608,7 @@ ProjectHook: - confidential_note_events - repository_update_events - releases_events +- emoji_events ProtectedBranch: - id - project_id diff --git a/spec/models/bulk_import_spec.rb b/spec/models/bulk_import_spec.rb index acb1f4a2ef7..a50fc6eaba4 100644 --- a/spec/models/bulk_import_spec.rb +++ b/spec/models/bulk_import_spec.rb @@ -75,4 +75,22 @@ RSpec.describe BulkImport, type: :model, feature_category: :importers do end end end + + describe '#supports_batched_export?' do + context 'when source version is greater than min supported version for batched migrations' do + it 'returns true' do + bulk_import = build(:bulk_import, source_version: '16.2.0') + + expect(bulk_import.supports_batched_export?).to eq(true) + end + end + + context 'when source version is less than min supported version for batched migrations' do + it 'returns false' do + bulk_import = build(:bulk_import, source_version: '15.5.0') + + expect(bulk_import.supports_batched_export?).to eq(false) + end + end + end end diff --git a/spec/models/bulk_imports/entity_spec.rb b/spec/models/bulk_imports/entity_spec.rb index c7ace3d2b78..7179ed7cb42 100644 --- a/spec/models/bulk_imports/entity_spec.rb +++ b/spec/models/bulk_imports/entity_spec.rb @@ -255,6 +255,27 @@ RSpec.describe BulkImports::Entity, type: :model, feature_category: :importers d expect(entity.export_relations_url_path).to eq("/projects/#{entity.source_xid}/export_relations") end end + + context 'when batched' do + context 'when source supports batched export' do + it 'returns batched export relations url' do + import = build(:bulk_import, source_version: '16.2.0') + entity = build(:bulk_import_entity, :project_entity, bulk_import: import) + + expect(entity.export_relations_url_path(batched: true)) + .to eq("/projects/#{entity.source_xid}/export_relations?batched=true") + end + end + + context 'when source does not support batched export' do + it 'returns export relations url' do + entity = build(:bulk_import_entity) + + expect(entity.export_relations_url_path(batched: true)) + .to eq("/groups/#{entity.source_xid}/export_relations") + end + end + end end describe '#relation_download_url_path' do @@ -264,6 +285,27 @@ RSpec.describe BulkImports::Entity, type: :model, feature_category: :importers d expect(entity.relation_download_url_path('test')) .to eq("/groups/#{entity.source_xid}/export_relations/download?relation=test") end + + context 'when batch number is present' do + context 'when source supports batched export' do + it 'returns export relations url with download query string and batch number' do + import = build(:bulk_import, source_version: '16.2.0') + entity = build(:bulk_import_entity, :project_entity, bulk_import: import) + + expect(entity.relation_download_url_path('test', 1)) + .to eq("/projects/#{entity.source_xid}/export_relations/download?batch_number=1&batched=true&relation=test") + end + end + + context 'when source does not support batched export' do + it 'returns export relations url' do + entity = build(:bulk_import_entity) + + expect(entity.relation_download_url_path('test', 1)) + .to eq("/groups/#{entity.source_xid}/export_relations/download?relation=test") + end + end + end end describe '#entity_type' do diff --git a/spec/models/bulk_imports/export_spec.rb b/spec/models/bulk_imports/export_spec.rb index 7173d032bc2..e5c0632b113 100644 --- a/spec/models/bulk_imports/export_spec.rb +++ b/spec/models/bulk_imports/export_spec.rb @@ -83,4 +83,29 @@ RSpec.describe BulkImports::Export, type: :model, feature_category: :importers d end end end + + describe '#remove_existing_upload!' do + context 'when upload exists' do + it 'removes the upload' do + export = create(:bulk_import_export) + upload = create(:bulk_import_export_upload, export: export) + upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz')) + + expect_any_instance_of(BulkImports::ExportUpload) do |upload| + expect(upload).to receive(:remove_export_file!) + expect(upload).to receive(:save!) + end + + export.remove_existing_upload! + end + end + + context 'when upload does not exist' do + it 'returns' do + export = build(:bulk_import_export) + + expect { export.remove_existing_upload! }.not_to change { export.upload } + end + end + end end diff --git a/spec/models/bulk_imports/export_status_spec.rb b/spec/models/bulk_imports/export_status_spec.rb index 0921c3bdce2..c3faa2db19c 100644 --- a/spec/models/bulk_imports/export_status_spec.rb +++ b/spec/models/bulk_imports/export_status_spec.rb @@ -2,16 +2,28 @@ require 'spec_helper' -RSpec.describe BulkImports::ExportStatus do +RSpec.describe BulkImports::ExportStatus, feature_category: :importers do let_it_be(:relation) { 'labels' } let_it_be(:import) { create(:bulk_import) } let_it_be(:config) { create(:bulk_import_configuration, bulk_import: import) } let_it_be(:entity) { create(:bulk_import_entity, bulk_import: import, source_full_path: 'foo') } let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let(:batched) { false } + let(:batches) { [] } let(:response_double) do instance_double(HTTParty::Response, - parsed_response: [{ 'relation' => 'labels', 'status' => status, 'error' => 'error!' }] + parsed_response: [ + { + 'relation' => 'labels', + 'status' => status, + 'error' => 'error!', + 'batched' => batched, + 'batches' => batches, + 'batches_count' => 1, + 'total_objects_count' => 1 + } + ] ) end @@ -190,4 +202,84 @@ RSpec.describe BulkImports::ExportStatus do end end end + + describe 'batching information' do + let(:status) { BulkImports::Export::FINISHED } + + describe '#batched?' do + context 'when export is batched' do + let(:batched) { true } + + it 'returns true' do + expect(subject.batched?).to eq(true) + end + end + + context 'when export is not batched' do + it 'returns false' do + expect(subject.batched?).to eq(false) + end + end + + context 'when export batch information is missing' do + let(:response_double) do + instance_double(HTTParty::Response, parsed_response: [{ 'relation' => 'labels', 'status' => status }]) + end + + it 'returns false' do + expect(subject.batched?).to eq(false) + end + end + end + + describe '#batches_count' do + context 'when batches count is present' do + it 'returns batches count' do + expect(subject.batches_count).to eq(1) + end + end + + context 'when batches count is missing' do + let(:response_double) do + instance_double(HTTParty::Response, parsed_response: [{ 'relation' => 'labels', 'status' => status }]) + end + + it 'returns 0' do + expect(subject.batches_count).to eq(0) + end + end + end + + describe '#batch' do + context 'when export is batched' do + let(:batched) { true } + let(:batches) do + [ + { 'relation' => 'labels', 'status' => status, 'batch_number' => 1 }, + { 'relation' => 'milestones', 'status' => status, 'batch_number' => 2 } + ] + end + + context 'when batch number is in range' do + it 'returns batch information' do + expect(subject.batch(1)['relation']).to eq('labels') + expect(subject.batch(2)['relation']).to eq('milestones') + expect(subject.batch(3)).to eq(nil) + end + end + end + + context 'when batch number is less than 1' do + it 'raises error' do + expect { subject.batch(0) }.to raise_error(ArgumentError) + end + end + + context 'when export is not batched' do + it 'returns nil' do + expect(subject.batch(1)).to eq(nil) + end + end + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 9d7f6674ab8..1a7a0688abe 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6394,6 +6394,14 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr expect(project.has_active_hooks?(:merge_request_hooks)).to eq(true) expect(project.has_active_hooks?).to eq(true) end + + context 'with :emoji_hooks scope' do + it 'returns true when a matching emoji hook exists' do + create(:project_hook, emoji_events: true, project: project) + + expect(project.has_active_hooks?(:emoji_hooks)).to eq(true) + end + end end describe '#has_active_integrations?' do diff --git a/spec/models/projects/triggered_hooks_spec.rb b/spec/models/projects/triggered_hooks_spec.rb index 3c885bdac8e..581ccb500e2 100644 --- a/spec/models/projects/triggered_hooks_spec.rb +++ b/spec/models/projects/triggered_hooks_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Projects::TriggeredHooks do +RSpec.describe Projects::TriggeredHooks, feature_category: :webhooks do let_it_be(:project) { create(:project) } let_it_be(:universal_push_hook) { create(:project_hook, project: project, push_events: true) } @@ -10,6 +10,7 @@ RSpec.describe Projects::TriggeredHooks do let_it_be(:issues_hook) { create(:project_hook, project: project, issues_events: true, push_events: false) } let(:wh_service) { instance_double(::WebHookService, async_execute: true) } + let(:data) { { some: 'data', as: 'json' } } def run_hooks(scope, data) hooks = described_class.new(scope, data) @@ -18,8 +19,6 @@ RSpec.describe Projects::TriggeredHooks do end it 'executes hooks by scope' do - data = { some: 'data', as: 'json' } - expect_hook_execution(issues_hook, data, 'issue_hooks') run_hooks(:issue_hooks, data) @@ -42,6 +41,40 @@ RSpec.describe Projects::TriggeredHooks do run_hooks(:push_hooks, data) end + context 'with emoji hooks' do + let_it_be(:emoji_hook) { create(:project_hook, project: project, emoji_events: true) } + + it 'executes hook' do + expect_hook_execution(emoji_hook, data, 'emoji_hooks') + + run_hooks(:emoji_hooks, data) + end + + context 'when emoji_webhooks feature flag is disabled' do + before do + stub_feature_flags(emoji_webhooks: false) + end + + it 'does not execute the hook' do + expect(WebHookService).not_to receive(:new) + + run_hooks(:emoji_hooks, data) + end + end + + context 'when emoji_webhooks feature flag is enabled for the project' do + before do + stub_feature_flags(emoji_webhooks: emoji_hook.project) + end + + it 'executes the hook' do + expect_hook_execution(emoji_hook, data, 'emoji_hooks') + + run_hooks(:emoji_hooks, data) + end + end + end + def expect_hook_execution(hook, data, scope) expect(WebHookService).to receive(:new).with(hook, data, scope).and_return(wh_service) end diff --git a/spec/requests/api/project_hooks_spec.rb b/spec/requests/api/project_hooks_spec.rb index c6bf77e5dcf..9d94b5437b7 100644 --- a/spec/requests/api/project_hooks_spec.rb +++ b/spec/requests/api/project_hooks_spec.rb @@ -57,6 +57,7 @@ RSpec.describe API::ProjectHooks, 'ProjectHooks', feature_category: :webhooks do job_events deployment_events releases_events + emoji_events ] end diff --git a/spec/services/award_emojis/add_service_spec.rb b/spec/services/award_emojis/add_service_spec.rb index 99dbe6dc606..e90ea284f29 100644 --- a/spec/services/award_emojis/add_service_spec.rb +++ b/spec/services/award_emojis/add_service_spec.rb @@ -53,6 +53,12 @@ RSpec.describe AwardEmojis::AddService, feature_category: :team_planning do expect(award.user).to eq(user) end + it 'executes hooks' do + expect(service).to receive(:execute_hooks).with(kind_of(AwardEmoji), 'award') + + service.execute + end + describe 'marking Todos as done' do subject { service.execute } diff --git a/spec/services/award_emojis/base_service_spec.rb b/spec/services/award_emojis/base_service_spec.rb index f1ee4d1cfb8..0f67c619a48 100644 --- a/spec/services/award_emojis/base_service_spec.rb +++ b/spec/services/award_emojis/base_service_spec.rb @@ -3,15 +3,17 @@ require 'spec_helper' RSpec.describe AwardEmojis::BaseService, feature_category: :team_planning do - let(:awardable) { build(:note) } - let(:current_user) { build(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:current_user) { create(:user) } + let_it_be_with_reload(:awardable) { create(:note, project: project) } + + let(:emoji_name) { 'horse' } describe '.initialize' do subject { described_class } it 'uses same emoji name if not an alias' do - emoji_name = 'horse' - expect(subject.new(awardable, emoji_name, current_user).name).to eq(emoji_name) end @@ -22,4 +24,31 @@ RSpec.describe AwardEmojis::BaseService, feature_category: :team_planning do expect(subject.new(awardable, emoji_alias, current_user).name).to eq(emoji_name) end end + + describe '.execute_hooks' do + let(:award_emoji) { create(:award_emoji, awardable: awardable) } + let(:action) { 'award' } + + subject { described_class.new(awardable, emoji_name, current_user) } + + context 'with no emoji hooks configured' do + it 'does not build hook_data' do + expect(Gitlab::DataBuilder::Emoji).not_to receive(:build) + expect(award_emoji.awardable.project).not_to receive(:execute_hooks) + + subject.execute_hooks(award_emoji, action) + end + end + + context 'with emoji hooks configured' do + it 'builds hook_data and calls execute_hooks for project' do + hook_data = {} + create(:project_hook, project: project, emoji_events: true) + expect(Gitlab::DataBuilder::Emoji).to receive(:build).and_return(hook_data) + expect(award_emoji.awardable.project).to receive(:execute_hooks).with(hook_data, :emoji_hooks) + + subject.execute_hooks(award_emoji, action) + end + end + end end diff --git a/spec/services/award_emojis/destroy_service_spec.rb b/spec/services/award_emojis/destroy_service_spec.rb index 109bdbfa986..fbadee87f45 100644 --- a/spec/services/award_emojis/destroy_service_spec.rb +++ b/spec/services/award_emojis/destroy_service_spec.rb @@ -85,6 +85,12 @@ RSpec.describe AwardEmojis::DestroyService, feature_category: :team_planning do expect(result[:award]).to eq(award_from_user) expect(result[:award]).to be_destroyed end + + it 'executes hooks' do + expect(service).to receive(:execute_hooks).with(award_from_user, 'revoke') + + service.execute + end end end end diff --git a/spec/services/test_hooks/project_service_spec.rb b/spec/services/test_hooks/project_service_spec.rb index 31f97edbd08..35c827d5448 100644 --- a/spec/services/test_hooks/project_service_spec.rb +++ b/spec/services/test_hooks/project_service_spec.rb @@ -207,5 +207,26 @@ RSpec.describe TestHooks::ProjectService, feature_category: :code_testing do expect(service.execute).to include(success_result) end end + + context 'emoji' do + let(:trigger) { 'emoji_events' } + let(:trigger_key) { :emoji_hooks } + + it 'returns error message if not enough data' do + expect(hook).not_to receive(:execute) + expect(service.execute).to have_attributes(status: :error, message: 'Ensure the project has notes.') + end + + it 'executes hook' do + note = create(:note) + allow(project).to receive_message_chain(:notes, :any?).and_return(true) + allow(project).to receive_message_chain(:notes, :last).and_return(note) + allow(Gitlab::DataBuilder::Emoji).to receive(:build).with(anything, current_user, 'award') + .and_return(sample_data) + + expect(hook).to receive(:execute).with(sample_data, trigger_key, force: true).and_return(success_result) + expect(service.execute).to include(success_result) + end + end end end diff --git a/spec/workers/bulk_imports/export_request_worker_spec.rb b/spec/workers/bulk_imports/export_request_worker_spec.rb index 2faa28ba489..0acc44c5cbf 100644 --- a/spec/workers/bulk_imports/export_request_worker_spec.rb +++ b/spec/workers/bulk_imports/export_request_worker_spec.rb @@ -112,6 +112,36 @@ RSpec.describe BulkImports::ExportRequestWorker, feature_category: :importers do it_behaves_like 'requests relations export for api resource' end + + context 'when source supports batched migration' do + let_it_be(:bulk_import) { create(:bulk_import, source_version: BulkImport.min_gl_version_for_migration_in_batches) } + let_it_be(:config) { create(:bulk_import_configuration, bulk_import: bulk_import) } + let_it_be(:entity) { create(:bulk_import_entity, :project_entity, source_full_path: 'foo/bar', bulk_import: bulk_import) } + + it 'requests relations export & schedules entity worker' do + expected_url = "/projects/#{entity.source_xid}/export_relations?batched=true" + + expect_next_instance_of(BulkImports::Clients::HTTP) do |client| + expect(client).to receive(:post).with(expected_url) + end + + described_class.new.perform(entity.id) + end + + context 'when bulk_imports_batched_import_export feature flag is disabled' do + it 'requests relation export without batched param' do + stub_feature_flags(bulk_imports_batched_import_export: false) + + expected_url = "/projects/#{entity.source_xid}/export_relations" + + expect_next_instance_of(BulkImports::Clients::HTTP) do |client| + expect(client).to receive(:post).with(expected_url) + end + + described_class.new.perform(entity.id) + end + end + end end describe '#sidekiq_retries_exhausted' do diff --git a/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb b/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb new file mode 100644 index 00000000000..6fe6b420f2b --- /dev/null +++ b/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::FinishBatchedPipelineWorker, feature_category: :importers do + let_it_be(:bulk_import) { create(:bulk_import) } + let_it_be(:config) { create(:bulk_import_configuration, bulk_import: bulk_import) } + let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) } + + let(:status_event) { :finish } + let(:pipeline_tracker) { create(:bulk_import_tracker, :started, :batched, entity: entity) } + + subject(:worker) { described_class.new } + + describe '#perform' do + it 'finishes pipeline and enqueues entity worker' do + expect(BulkImports::EntityWorker) + .to receive(:perform_async) + .with(entity.id, pipeline_tracker.stage) + + subject.perform(pipeline_tracker.id) + + expect(pipeline_tracker.reload.finished?).to eq(true) + end + + context 'when import is in progress' do + it 're-enqueues' do + create(:bulk_import_batch_tracker, :started, tracker: pipeline_tracker) + + expect(described_class) + .to receive(:perform_in) + .with(described_class::REQUEUE_DELAY, pipeline_tracker.id) + + subject.perform(pipeline_tracker.id) + end + end + + context 'when pipeline tracker is stale' do + let(:pipeline_tracker) { create(:bulk_import_tracker, :started, :batched, :stale, entity: entity) } + + it 'fails pipeline tracker and its batches' do + create(:bulk_import_batch_tracker, :finished, tracker: pipeline_tracker) + + subject.perform(pipeline_tracker.id) + + expect(pipeline_tracker.reload.failed?).to eq(true) + expect(pipeline_tracker.batches.first.reload.failed?).to eq(true) + end + end + + context 'when pipeline is not batched' do + let(:pipeline_tracker) { create(:bulk_import_tracker, :started, entity: entity) } + + it 'returns' do + expect_next_instance_of(BulkImports::Tracker) do |instance| + expect(instance).not_to receive(:finish!) + end + + subject.perform(pipeline_tracker.id) + end + end + + context 'when pipeline is not started' do + let(:status_event) { :start } + + it 'returns' do + expect_next_instance_of(BulkImports::Tracker) do |instance| + expect(instance).not_to receive(:finish!) + end + + described_class.new.perform(pipeline_tracker.id) + end + end + end +end diff --git a/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb b/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb new file mode 100644 index 00000000000..c10e1b486ab --- /dev/null +++ b/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::PipelineBatchWorker, feature_category: :importers do + let_it_be(:bulk_import) { create(:bulk_import) } + let_it_be(:config) { create(:bulk_import_configuration, bulk_import: bulk_import) } + let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) } + + let(:pipeline_class) do + Class.new do + def initialize(_); end + + def run; end + + def self.file_extraction_pipeline? + false + end + end + end + + let(:tracker) do + create( + :bulk_import_tracker, + entity: entity, + pipeline_name: 'FakePipeline', + status_event: 'enqueue' + ) + end + + let(:batch) { create(:bulk_import_batch_tracker, :created, tracker: tracker) } + + subject(:worker) { described_class.new } + + before do + stub_const('FakePipeline', pipeline_class) + + allow(subject).to receive(:jid).and_return('jid') + allow(entity).to receive(:pipeline_exists?).with('FakePipeline').and_return(true) + allow_next_instance_of(BulkImports::Groups::Stage) do |instance| + allow(instance) + .to receive(:pipelines) + .and_return([{ stage: 0, pipeline: pipeline_class }]) + end + end + + describe '#perform' do + it 'runs the given pipeline batch successfully' do + expect(BulkImports::FinishBatchedPipelineWorker).to receive(:perform_async).with(tracker.id) + + subject.perform(batch.id) + + expect(batch.reload).to be_finished + end + + context 'when tracker is failed' do + let(:tracker) { create(:bulk_import_tracker, :failed) } + + it 'skips the batch' do + subject.perform(batch.id) + + expect(batch.reload).to be_skipped + end + end + + context 'when tracker is finished' do + let(:tracker) { create(:bulk_import_tracker, :finished) } + + it 'skips the batch' do + subject.perform(batch.id) + + expect(batch.reload).to be_skipped + end + end + + context 'when exclusive lease cannot be obtained' do + it 'does not run the pipeline' do + expect(subject).to receive(:try_obtain_lease).and_return(false) + expect(subject).not_to receive(:run) + + subject.perform(batch.id) + end + end + + context 'when pipeline raises an exception' do + context 'when pipeline is retryable' do + it 'retries the batch' do + allow_next_instance_of(pipeline_class) do |instance| + allow(instance) + .to receive(:run) + .and_raise(BulkImports::RetryPipelineError.new('Error!', 60)) + end + + expect(described_class).to receive(:perform_in).with(60, batch.id) + + subject.perform(batch.id) + + expect(batch.reload).to be_created + end + end + + context 'when pipeline is not retryable' do + it 'fails the batch and creates a failure record' do + allow_next_instance_of(pipeline_class) do |instance| + allow(instance).to receive(:run).and_raise(StandardError, 'Something went wrong') + end + + expect(Gitlab::ErrorTracking).to receive(:track_exception).with( + instance_of(StandardError), + hash_including( + batch_id: batch.id, + tracker_id: tracker.id, + pipeline_class: 'FakePipeline', + pipeline_step: 'pipeline_batch_worker_run' + ) + ) + + expect(BulkImports::Failure).to receive(:create).with( + bulk_import_entity_id: entity.id, + pipeline_class: 'FakePipeline', + pipeline_step: 'pipeline_batch_worker_run', + exception_class: 'StandardError', + exception_message: 'Something went wrong', + correlation_id_value: anything + ) + + expect(BulkImports::FinishBatchedPipelineWorker).to receive(:perform_async).with(tracker.id) + + subject.perform(batch.id) + + expect(batch.reload).to be_failed + end + end + end + end +end diff --git a/spec/workers/bulk_imports/pipeline_worker_spec.rb b/spec/workers/bulk_imports/pipeline_worker_spec.rb index e8b0714471d..320f62dc93e 100644 --- a/spec/workers/bulk_imports/pipeline_worker_spec.rb +++ b/spec/workers/bulk_imports/pipeline_worker_spec.rb @@ -387,8 +387,9 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do stub_const('NdjsonPipeline', file_extraction_pipeline) allow_next_instance_of(BulkImports::Groups::Stage) do |instance| - allow(instance).to receive(:pipelines) - .and_return([{ stage: 0, pipeline: file_extraction_pipeline }]) + allow(instance) + .to receive(:pipelines) + .and_return([{ stage: 0, pipeline: file_extraction_pipeline }]) end end @@ -397,6 +398,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do allow(status).to receive(:started?).and_return(false) allow(status).to receive(:empty?).and_return(false) allow(status).to receive(:failed?).and_return(false) + allow(status).to receive(:batched?).and_return(false) end subject.perform(pipeline_tracker.id, pipeline_tracker.stage, entity.id) @@ -410,6 +412,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do allow(status).to receive(:started?).and_return(true) allow(status).to receive(:empty?).and_return(false) allow(status).to receive(:failed?).and_return(false) + allow(status).to receive(:batched?).and_return(false) end expect(described_class) @@ -431,6 +434,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do allow(status).to receive(:started?).and_return(false) allow(status).to receive(:empty?).and_return(true) allow(status).to receive(:failed?).and_return(false) + allow(status).to receive(:batched?).and_return(false) end pipeline_tracker.update!(created_at: created_at) @@ -572,5 +576,43 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do expect(pipeline_tracker.reload.status_name).to eq(:failed) end end + + context 'when export is batched' do + let(:batches_count) { 2 } + + before do + allow_next_instance_of(BulkImports::ExportStatus) do |status| + allow(status).to receive(:batched?).and_return(true) + allow(status).to receive(:batches_count).and_return(batches_count) + allow(status).to receive(:started?).and_return(false) + allow(status).to receive(:empty?).and_return(false) + allow(status).to receive(:failed?).and_return(false) + end + end + + it 'enqueues pipeline batches' do + expect(BulkImports::PipelineBatchWorker).to receive(:perform_async).twice + + subject.perform(pipeline_tracker.id, pipeline_tracker.stage, entity.id) + + pipeline_tracker.reload + + expect(pipeline_tracker.status_name).to eq(:started) + expect(pipeline_tracker.batched).to eq(true) + expect(pipeline_tracker.batches.count).to eq(batches_count) + end + + context 'when batches count is less than 1' do + let(:batches_count) { 0 } + + it 'marks tracker as finished' do + expect(subject).not_to receive(:enqueue_batches) + + subject.perform(pipeline_tracker.id, pipeline_tracker.stage, entity.id) + + expect(pipeline_tracker.reload.status_name).to eq(:finished) + end + end + end end end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index a716cc4d012..38959b6d764 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -141,6 +141,7 @@ RSpec.describe 'Every Sidekiq worker', feature_category: :shared do 'BulkImports::ExportRequestWorker' => 5, 'BulkImports::EntityWorker' => false, 'BulkImports::PipelineWorker' => false, + 'BulkImports::PipelineBatchWorker' => false, 'Chaos::CpuSpinWorker' => 3, 'Chaos::DbSpinWorker' => 3, 'Chaos::KillWorker' => false, |