diff options
41 files changed, 723 insertions, 86 deletions
diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml index 73d6c2f21af..500ba1e7544 100644 --- a/.rubocop_manual_todo.yml +++ b/.rubocop_manual_todo.yml @@ -2557,7 +2557,6 @@ Rails/IncludeUrlHelper: # TODO issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344279 Style/OpenStructUse: Exclude: - - 'app/finders/snippets_finder.rb' - 'app/helpers/application_settings_helper.rb' - 'ee/spec/db/production/license_spec.rb' - 'ee/spec/features/projects/new_project_spec.rb' diff --git a/app/controllers/projects/tags_controller.rb b/app/controllers/projects/tags_controller.rb index 581680f1b13..00b0b791f01 100644 --- a/app/controllers/projects/tags_controller.rb +++ b/app/controllers/projects/tags_controller.rb @@ -14,18 +14,24 @@ class Projects::TagsController < Projects::ApplicationController # rubocop: disable CodeReuse/ActiveRecord def index - params[:sort] = params[:sort].presence || sort_value_recently_updated + begin + params[:sort] = params[:sort].presence || sort_value_recently_updated - @sort = params[:sort] + @sort = params[:sort] - @tags, @tags_loading_error = TagsFinder.new(@repository, params).execute + @tags = TagsFinder.new(@repository, params).execute - @tags = Kaminari.paginate_array(@tags).page(params[:page]) - tag_names = @tags.map(&:name) - @tags_pipelines = @project.ci_pipelines.latest_successful_for_refs(tag_names) + @tags = Kaminari.paginate_array(@tags).page(params[:page]) + tag_names = @tags.map(&:name) + @tags_pipelines = @project.ci_pipelines.latest_successful_for_refs(tag_names) - @releases = project.releases.where(tag: tag_names) - @tag_pipeline_statuses = Ci::CommitStatusesFinder.new(@project, @repository, current_user, @tags).execute + @releases = project.releases.where(tag: tag_names) + @tag_pipeline_statuses = Ci::CommitStatusesFinder.new(@project, @repository, current_user, @tags).execute + + rescue Gitlab::Git::CommandError => e + @tags = [] + @tags_loading_error = e + end respond_to do |format| status = @tags_loading_error ? :service_unavailable : :ok diff --git a/app/controllers/projects/tree_controller.rb b/app/controllers/projects/tree_controller.rb index a76d45411dd..72bb03b63a3 100644 --- a/app/controllers/projects/tree_controller.rb +++ b/app/controllers/projects/tree_controller.rb @@ -19,6 +19,7 @@ class Projects::TreeController < Projects::ApplicationController push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml) push_frontend_feature_flag(:paginated_tree_graphql_query, @project, default_enabled: :yaml) push_frontend_feature_flag(:new_dir_modal, @project, default_enabled: :yaml) + push_frontend_feature_flag(:refactor_blob_viewer, @project, default_enabled: :yaml) end feature_category :source_code_management diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 9dcd3984011..bbf33865a26 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -293,7 +293,11 @@ class ProjectsController < Projects::ApplicationController end if find_tags && @repository.tag_count.nonzero? - tags, _ = TagsFinder.new(@repository, params).execute + tags = begin + TagsFinder.new(@repository, params).execute + rescue Gitlab::Git::CommandError + [] + end options['Tags'] = tags.take(100).map(&:name) end diff --git a/app/finders/snippets_finder.rb b/app/finders/snippets_finder.rb index 81643826782..b1e12721712 100644 --- a/app/finders/snippets_finder.rb +++ b/app/finders/snippets_finder.rb @@ -42,12 +42,11 @@ class SnippetsFinder < UnionFinder include FinderMethods include Gitlab::Utils::StrongMemoize - attr_accessor :current_user, :params - delegate :explore, :only_personal, :only_project, :scope, :sort, to: :params + attr_reader :current_user, :params def initialize(current_user = nil, params = {}) @current_user = current_user - @params = OpenStruct.new(params) + @params = params if project && author raise( @@ -77,9 +76,9 @@ class SnippetsFinder < UnionFinder private def init_collection - if explore + if explore? snippets_for_explore - elsif only_personal + elsif only_personal? personal_snippets elsif project snippets_for_a_single_project @@ -110,7 +109,7 @@ class SnippetsFinder < UnionFinder # over the resulting SQL query. def snippets_for_personal_and_multiple_projects queries = [] - queries << personal_snippets unless only_project + queries << personal_snippets unless only_project? if Ability.allowed?(current_user, :read_cross_project) queries << snippets_of_visible_projects @@ -171,7 +170,7 @@ class SnippetsFinder < UnionFinder end def visibility_from_scope - case scope.to_s + case params[:scope].to_s when 'are_private' Snippet::PRIVATE when 'are_internal' @@ -206,7 +205,19 @@ class SnippetsFinder < UnionFinder end def sort_param - sort.presence || 'id_desc' + params[:sort].presence || 'id_desc' + end + + def explore? + params[:explore].present? + end + + def only_personal? + params[:only_personal].present? + end + + def only_project? + params[:only_project].present? end def prepared_union(queries) diff --git a/app/finders/tags_finder.rb b/app/finders/tags_finder.rb index 0ccbbdc1b87..36ca2c7f281 100644 --- a/app/finders/tags_finder.rb +++ b/app/finders/tags_finder.rb @@ -8,8 +8,6 @@ class TagsFinder < GitRefsFinder def execute tags = repository.tags_sorted_by(sort) - [by_search(tags), nil] - rescue Gitlab::Git::CommandError => e - [[], e] + by_search(tags) end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 14a0c38dcf0..9565d1a6a69 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -17,6 +17,9 @@ class Namespace < ApplicationRecord include EachBatch ignore_column :delayed_project_removal, remove_with: '14.1', remove_after: '2021-05-22' + # Temporary column used for back-filling project namespaces. + # Remove it once the back-filling of all project namespaces is done. + ignore_column :tmp_project_id, remove_with: '14.7', remove_after: '2022-01-22' # Tells ActiveRecord not to store the full class name, in order to save some space # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794 diff --git a/app/views/admin/users/_users.html.haml b/app/views/admin/users/_users.html.haml index 1a43d91b800..e62e4cfa192 100644 --- a/app/views/admin/users/_users.html.haml +++ b/app/views/admin/users/_users.html.haml @@ -3,48 +3,38 @@ = sprite_icon('chevron-lg-left', size: 12) .fade-right = sprite_icon('chevron-lg-right', size: 12) - %ul.nav-links.nav.nav-tabs.scrolling-tabs - = nav_link(html_options: { class: active_when(params[:filter].nil?) }) do - = link_to admin_users_path do - = s_('AdminUsers|Active') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.active_without_ghosts) - = nav_link(html_options: { class: active_when(params[:filter] == 'admins') }) do - = link_to admin_users_path(filter: "admins") do - = s_('AdminUsers|Admins') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.admins) - = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_enabled')} filter-two-factor-enabled" }) do - = link_to admin_users_path(filter: 'two_factor_enabled') do - = s_('AdminUsers|2FA Enabled') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.with_two_factor) - = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_disabled')} filter-two-factor-disabled" }) do - = link_to admin_users_path(filter: 'two_factor_disabled') do - = s_('AdminUsers|2FA Disabled') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.without_two_factor) - = nav_link(html_options: { class: active_when(params[:filter] == 'external') }) do - = link_to admin_users_path(filter: 'external') do - = s_('AdminUsers|External') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.external) - = nav_link(html_options: { class: active_when(params[:filter] == 'blocked') }) do - = link_to admin_users_path(filter: "blocked") do - = s_('AdminUsers|Blocked') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.blocked) + = gl_tabs_nav({ class: 'scrolling-tabs nav-links gl-display-flex gl-flex-grow-1' }) do + = gl_tab_link_to admin_users_path, { item_active: active_when(params[:filter].nil?), class: 'gl-border-0!' } do + = s_('AdminUsers|Active') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.active_without_ghosts)) + = gl_tab_link_to admin_users_path(filter: "admins"), { item_active: active_when(params[:filter] == 'admins'), class: 'gl-border-0!' } do + = s_('AdminUsers|Admins') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.admins)) + = gl_tab_link_to admin_users_path(filter: 'two_factor_enabled'), { item_active: active_when(params[:filter] == 'two_factor_enabled'), class: 'filter-two-factor-enabled gl-border-0!' } do + = s_('AdminUsers|2FA Enabled') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.with_two_factor)) + = gl_tab_link_to admin_users_path(filter: 'two_factor_disabled'), { item_active: active_when(params[:filter] == 'two_factor_disabled'), class: 'filter-two-factor-disabled gl-border-0!' } do + = s_('AdminUsers|2FA Disabled') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.without_two_factor)) + = gl_tab_link_to admin_users_path(filter: 'external'), { item_active: active_when(params[:filter] == 'external'), class: 'gl-border-0!' } do + = s_('AdminUsers|External') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.external)) + = gl_tab_link_to admin_users_path(filter: "blocked"), { item_active: active_when(params[:filter] == 'blocked'), class: 'gl-border-0!' } do + = s_('AdminUsers|Blocked') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.blocked)) - if ban_feature_available? - = nav_link(html_options: { class: active_when(params[:filter] == 'banned') }) do - = link_to admin_users_path(filter: "banned") do - = s_('AdminUsers|Banned') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.banned) - = nav_link(html_options: { class: "#{active_when(params[:filter] == 'blocked_pending_approval')} filter-blocked-pending-approval" }) do - = link_to admin_users_path(filter: "blocked_pending_approval"), data: { qa_selector: 'pending_approval_tab' } do - = s_('AdminUsers|Pending approval') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.blocked_pending_approval) - = nav_link(html_options: { class: active_when(params[:filter] == 'deactivated') }) do - = link_to admin_users_path(filter: "deactivated") do - = s_('AdminUsers|Deactivated') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.deactivated) - = nav_link(html_options: { class: active_when(params[:filter] == 'wop') }) do - = link_to admin_users_path(filter: "wop") do - = s_('AdminUsers|Without projects') - %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.without_projects) + = gl_tab_link_to admin_users_path(filter: "banned"), { item_active: active_when(params[:filter] == 'banned'), class: 'gl-border-0!' } do + = s_('AdminUsers|Banned') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.banned)) + = gl_tab_link_to admin_users_path(filter: "blocked_pending_approval"), { item_active: active_when(params[:filter] == 'blocked_pending_approval'), class: 'filter-blocked-pending-approval gl-border-0!', data: { qa_selector: 'pending_approval_tab' } } do + = s_('AdminUsers|Pending approval') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.blocked_pending_approval)) + = gl_tab_link_to admin_users_path(filter: "deactivated"), { item_active: active_when(params[:filter] == 'deactivated'), class: 'gl-border-0!' } do + = s_('AdminUsers|Deactivated') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.deactivated)) + = gl_tab_link_to admin_users_path(filter: "wop"), { item_active: active_when(params[:filter] == 'wop'), class: 'gl-border-0!' } do + = s_('AdminUsers|Without projects') + = gl_tab_counter_badge(limited_counter_with_delimiter(User.without_projects)) .nav-controls = render_if_exists 'admin/users/admin_email_users' = render_if_exists 'admin/users/admin_export_user_permissions' diff --git a/app/views/projects/blame/show.html.haml b/app/views/projects/blame/show.html.haml index 30c052e054f..704576619a7 100644 --- a/app/views/projects/blame/show.html.haml +++ b/app/views/projects/blame/show.html.haml @@ -20,7 +20,7 @@ %span.legend-box.legend-box-9 %span.right-label Older - .table-responsive.file-content.blame.code.js-syntax-highlight + .table-responsive.file-content.blame.code{ class: user_color_scheme } %table - current_line = 1 - @blame.groups.each do |blame_group| diff --git a/config/feature_flags/development/jupyter_clean_diffs.yml b/config/feature_flags/development/jupyter_clean_diffs.yml index 4608cab91cf..0f3f6fe3057 100644 --- a/config/feature_flags/development/jupyter_clean_diffs.yml +++ b/config/feature_flags/development/jupyter_clean_diffs.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/343433 milestone: '14.5' type: development group: group::incubation -default_enabled: false +default_enabled: true diff --git a/data/deprecations/14-5-deprecation-vsa-announce-deprecation-of-vsa-filtering-calculation.yml b/data/deprecations/14-5-deprecation-vsa-announce-deprecation-of-vsa-filtering-calculation.yml new file mode 100644 index 00000000000..2961fab7487 --- /dev/null +++ b/data/deprecations/14-5-deprecation-vsa-announce-deprecation-of-vsa-filtering-calculation.yml @@ -0,0 +1,13 @@ +- name: "Value Stream Analytics filtering calculation change" # The name of the feature to be deprecated + announcement_milestone: "14.5" # The milestone when this feature was first announced as deprecated. + announcement_date: "2021-11-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post. + removal_milestone: "15.0" # The milestone when this feature is planned to be removed + body: | # Do not modify this line, instead modify the lines below. + We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. + + If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change. + stage: manage # (optional - may be required in the future) String value of the stage that the feature was created in. e.g., Growth + tiers: # (optional - may be required in the future) An array of tiers that the feature is available in currently. e.g., [Free, Silver, Gold, Core, Premium, Ultimate] + issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/343210' # (optional) This is a link to the deprecation issue in GitLab + documentation_url: "https://docs.gitlab.com/ee/user/analytics/value_stream_analytics.html#filter-value-stream-analytics-data" # (optional) This is a link to the current documentation page + image_url: "vsa_warning.png" # (optional) This is a link to a thumbnail image depicting the feature diff --git a/data/deprecations/vsa_warning.png b/data/deprecations/vsa_warning.png Binary files differnew file mode 100644 index 00000000000..7ee6c1e2343 --- /dev/null +++ b/data/deprecations/vsa_warning.png diff --git a/db/migrate/20211018101034_add_tmp_project_id_column_to_namespaces.rb b/db/migrate/20211018101034_add_tmp_project_id_column_to_namespaces.rb new file mode 100644 index 00000000000..cc73cb5047b --- /dev/null +++ b/db/migrate/20211018101034_add_tmp_project_id_column_to_namespaces.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class AddTmpProjectIdColumnToNamespaces < Gitlab::Database::Migration[1.0] + enable_lock_retries! + + def change + # this is a temporary column to be able to batch insert records into namespaces table and then be able to link these + # to projects table. + add_column :namespaces, :tmp_project_id, :integer # rubocop: disable Migration/AddColumnsToWideTables + end +end diff --git a/db/post_migrate/20211018101552_add_index_to_tmp_project_id_column_on_namespaces_table.rb b/db/post_migrate/20211018101552_add_index_to_tmp_project_id_column_on_namespaces_table.rb new file mode 100644 index 00000000000..b50b7fa21a5 --- /dev/null +++ b/db/post_migrate/20211018101552_add_index_to_tmp_project_id_column_on_namespaces_table.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexToTmpProjectIdColumnOnNamespacesTable < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_index_on_tmp_project_id_on_namespaces' + + def up + add_concurrent_index :namespaces, :tmp_project_id, name: INDEX_NAME, unique: true + end + + def down + remove_concurrent_index_by_name :namespaces, INDEX_NAME + end +end diff --git a/db/post_migrate/20211018101852_add_fk_to_tmp_project_id_column_on_namespaces_table.rb b/db/post_migrate/20211018101852_add_fk_to_tmp_project_id_column_on_namespaces_table.rb new file mode 100644 index 00000000000..f2772389d02 --- /dev/null +++ b/db/post_migrate/20211018101852_add_fk_to_tmp_project_id_column_on_namespaces_table.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddFkToTmpProjectIdColumnOnNamespacesTable < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :namespaces, :projects, column: :tmp_project_id + end + + def down + remove_foreign_key :namespaces, column: :tmp_project_id + end +end diff --git a/db/post_migrate/20211018102252_add_index_to_group_id_column_on_webhooks_table.rb b/db/post_migrate/20211018102252_add_index_to_group_id_column_on_webhooks_table.rb new file mode 100644 index 00000000000..d5fac373ca4 --- /dev/null +++ b/db/post_migrate/20211018102252_add_index_to_group_id_column_on_webhooks_table.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexToGroupIdColumnOnWebhooksTable < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + INDEX_NAME = 'index_on_group_id_on_webhooks' + + def up + add_concurrent_index :web_hooks, :group_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :web_hooks, INDEX_NAME + end +end diff --git a/db/schema_migrations/20211018101034 b/db/schema_migrations/20211018101034 new file mode 100644 index 00000000000..57b1cc1e3c2 --- /dev/null +++ b/db/schema_migrations/20211018101034 @@ -0,0 +1 @@ +1cadc3a932d5b62cfeafcd4090eddc37b44997dbbd0b34da1c7c87a5774bb683
\ No newline at end of file diff --git a/db/schema_migrations/20211018101552 b/db/schema_migrations/20211018101552 new file mode 100644 index 00000000000..3814122d9a3 --- /dev/null +++ b/db/schema_migrations/20211018101552 @@ -0,0 +1 @@ +9a62f0ec43ab295619d82494090c38539cb16408c8971bdde86bb8d02546f558
\ No newline at end of file diff --git a/db/schema_migrations/20211018101852 b/db/schema_migrations/20211018101852 new file mode 100644 index 00000000000..5288e753211 --- /dev/null +++ b/db/schema_migrations/20211018101852 @@ -0,0 +1 @@ +30e9632877d3ad33528be0f56962c0ab57f5eee3889183d9638cbaea903a3d82
\ No newline at end of file diff --git a/db/schema_migrations/20211018102252 b/db/schema_migrations/20211018102252 new file mode 100644 index 00000000000..7d8b2c19da1 --- /dev/null +++ b/db/schema_migrations/20211018102252 @@ -0,0 +1 @@ +14bb815cbdad2db56dafb7eaaff893de96116a1a9e8d6c5ed95f4bef9b9717fc
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index f1c6725d927..e0efa8b9c39 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -16375,7 +16375,8 @@ CREATE TABLE namespaces ( push_rule_id bigint, shared_runners_enabled boolean DEFAULT true NOT NULL, allow_descendants_override_disabled_shared_runners boolean DEFAULT false NOT NULL, - traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL, + tmp_project_id integer ); CREATE SEQUENCE namespaces_id_seq @@ -26591,6 +26592,8 @@ CREATE INDEX index_oauth_openid_requests_on_access_grant_id ON oauth_openid_requ CREATE UNIQUE INDEX index_on_deploy_keys_id_and_type_and_public ON keys USING btree (id, type) WHERE (public = true); +CREATE INDEX index_on_group_id_on_webhooks ON web_hooks USING btree (group_id); + CREATE INDEX index_on_identities_lower_extern_uid_and_provider ON identities USING btree (lower((extern_uid)::text), provider); CREATE UNIQUE INDEX index_on_instance_statistics_recorded_at_and_identifier ON analytics_usage_trends_measurements USING btree (identifier, recorded_at); @@ -27769,6 +27772,8 @@ CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_child_namespaces ON n CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_root_namespaces ON namespaces USING btree (id) WHERE ((parent_id IS NULL) AND (traversal_ids = '{}'::integer[])); +CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USING btree (tmp_project_id); + CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2); CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name); @@ -29012,6 +29017,9 @@ ALTER TABLE ONLY application_settings ALTER TABLE ONLY merge_requests ADD CONSTRAINT fk_6a5165a692 FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; +ALTER TABLE ONLY namespaces + ADD CONSTRAINT fk_6a77f66919 FOREIGN KEY (tmp_project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY geo_event_log ADD CONSTRAINT fk_6ada82d42a FOREIGN KEY (container_repository_updated_event_id) REFERENCES geo_container_repository_updated_events(id) ON DELETE CASCADE; diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 027e0568219..35dcc57a0d4 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -157,6 +157,14 @@ In milestone 15.0, we will remove the feature flag entirely. Moving forward, you Announced: 2021-11-22 +### Value Stream Analytics filtering calculation change + +We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. + +If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change. + +Announced: 2021-11-22 + ### `AuthenticationType` for `[runners.cache.s3]` must be explicitly assigned In GitLab 15.0 and later, to access the AWS S3 cache, you must specify the `AuthenticationType` for [`[runners.cache.s3]`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscaches3-section). The `AuthenticationType` must be `IAM` or `credentials`. diff --git a/lib/api/tags.rb b/lib/api/tags.rb index f018b421edd..9bcb1503194 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -24,7 +24,7 @@ module API use :pagination end get ':id/repository/tags', feature_category: :source_code_management do - tags, _ = ::TagsFinder.new(user_project.repository, + tags = ::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}", search: params[:search]).execute @@ -35,6 +35,9 @@ module API else present paginated_tags, with: Entities::Tag, project: user_project end + + rescue Gitlab::Git::CommandError + service_unavailable! end desc 'Get a single repository tag' do diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb new file mode 100644 index 00000000000..8e94c16369e --- /dev/null +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module ProjectNamespaces + # Back-fill project namespaces for projects that do not yet have a namespace. + # + # TODO: remove this comment when an actuall backfill migration is added. + # + # This is first being added without an actual migration as we need to initially test + # if backfilling project namespaces affects performance in any significant way. + # rubocop: disable Metrics/ClassLength + class BackfillProjectNamespaces + BATCH_SIZE = 100 + DELETE_BATCH_SIZE = 10 + PROJECT_NAMESPACE_STI_NAME = 'Project' + + IsolatedModels = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models + + def perform(start_id, end_id, namespace_id, migration_type = 'up') + load_project_ids(start_id, end_id, namespace_id) + + case migration_type + when 'up' + backfill_project_namespaces(namespace_id) + mark_job_as_succeeded(start_id, end_id, namespace_id, 'up') + when 'down' + cleanup_backfilled_project_namespaces(namespace_id) + mark_job_as_succeeded(start_id, end_id, namespace_id, 'down') + else + raise "Unknown migration type" + end + end + + private + + attr_accessor :project_ids + + def backfill_project_namespaces(namespace_id) + project_ids.each_slice(BATCH_SIZE) do |project_ids| + # We need to lock these project records for the period when we create project namespaces + # and link them to projects so that if a project is modified in the time between creating + # project namespaces `batch_insert_namespaces` and linking them to projects `batch_update_projects` + # we do not get them out of sync. + # + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72527#note_730679469 + Project.transaction do + Project.where(id: project_ids).select(:id).lock!('FOR UPDATE') + + batch_insert_namespaces(project_ids) + batch_update_projects(project_ids) + end + + batch_update_project_namespaces_traversal_ids(project_ids) + end + end + + def cleanup_backfilled_project_namespaces(namespace_id) + project_ids.each_slice(BATCH_SIZE) do |project_ids| + # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records + # from namespaces are deleted due to FK/triggers + nullify_project_namespaces_in_projects(project_ids) + delete_project_namespace_records(project_ids) + end + end + + def batch_insert_namespaces(project_ids) + projects = IsolatedModels::Project.where(id: project_ids) + .select("projects.id, projects.name, projects.path, projects.namespace_id, projects.visibility_level, shared_runners_enabled, '#{PROJECT_NAMESPACE_STI_NAME}', now(), now()") + + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO namespaces (tmp_project_id, name, path, parent_id, visibility_level, shared_runners_enabled, type, created_at, updated_at) + #{projects.to_sql} + ON CONFLICT DO NOTHING; + SQL + end + + def batch_update_projects(project_ids) + projects = IsolatedModels::Project.where(id: project_ids) + .joins("INNER JOIN namespaces ON projects.id = namespaces.tmp_project_id") + .select("namespaces.id, namespaces.tmp_project_id") + + ActiveRecord::Base.connection.execute <<~SQL + WITH cte(project_namespace_id, project_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{projects.to_sql} + ) + UPDATE projects + SET project_namespace_id = cte.project_namespace_id + FROM cte + WHERE id = cte.project_id AND projects.project_namespace_id IS DISTINCT FROM cte.project_namespace_id + SQL + end + + def batch_update_project_namespaces_traversal_ids(project_ids) + namespaces = Namespace.where(tmp_project_id: project_ids) + .joins("INNER JOIN namespaces n2 ON namespaces.parent_id = n2.id") + .select("namespaces.id as project_namespace_id, n2.traversal_ids") + + ActiveRecord::Base.connection.execute <<~SQL + UPDATE namespaces + SET traversal_ids = array_append(project_namespaces.traversal_ids, project_namespaces.project_namespace_id) + FROM (#{namespaces.to_sql}) as project_namespaces(project_namespace_id, traversal_ids) + WHERE id = project_namespaces.project_namespace_id + SQL + end + + def nullify_project_namespaces_in_projects(project_ids) + IsolatedModels::Project.where(id: project_ids).update_all(project_namespace_id: nil) + end + + def delete_project_namespace_records(project_ids) + project_ids.each_slice(DELETE_BATCH_SIZE) do |p_ids| + IsolatedModels::Namespace.where(type: PROJECT_NAMESPACE_STI_NAME).where(tmp_project_id: p_ids).delete_all + end + end + + def load_project_ids(start_id, end_id, namespace_id) + projects = IsolatedModels::Project.arel_table + relation = IsolatedModels::Project.where(projects[:id].between(start_id..end_id)) + relation = relation.where(projects[:namespace_id].in(Arel::Nodes::SqlLiteral.new(hierarchy_cte(namespace_id)))) if namespace_id + + @project_ids = relation.pluck(:id) + end + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('BackfillProjectNamespaces', arguments) + end + + def hierarchy_cte(root_namespace_id) + <<-SQL + WITH RECURSIVE "base_and_descendants" AS ( + ( + SELECT "namespaces"."id" + FROM "namespaces" + WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = #{root_namespace_id.to_i} + ) + UNION + ( + SELECT "namespaces"."id" + FROM "namespaces", "base_and_descendants" + WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id" + ) + ) + SELECT "id" FROM "base_and_descendants" AS "namespaces" + SQL + end + end + # rubocop: enable Metrics/ClassLength + end + end +end diff --git a/lib/gitlab/background_migration/project_namespaces/models/namespace.rb b/lib/gitlab/background_migration/project_namespaces/models/namespace.rb new file mode 100644 index 00000000000..5576c34cf65 --- /dev/null +++ b/lib/gitlab/background_migration/project_namespaces/models/namespace.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module ProjectNamespaces + module Models + # isolated Namespace model + class Namespace < ActiveRecord::Base + include EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + end + end + end + end +end diff --git a/lib/gitlab/background_migration/project_namespaces/models/project.rb b/lib/gitlab/background_migration/project_namespaces/models/project.rb new file mode 100644 index 00000000000..4a6a309e289 --- /dev/null +++ b/lib/gitlab/background_migration/project_namespaces/models/project.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module ProjectNamespaces + module Models + # isolated Project model + class Project < ActiveRecord::Base + include EachBatch + + self.table_name = 'projects' + end + end + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index f96564f111f..708647f2b10 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,7 +44,7 @@ module Gitlab new_blob_lazy old_blob_lazy - preprocess_before_diff(diff) if Feature.enabled?(:jupyter_clean_diffs, repository.project) + preprocess_before_diff(diff) if Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) end def position(position_marker, position_type: :text) diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b768011335e..aedcfe3cb40 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -153,7 +153,7 @@ module Gitlab blob.load_all_data! - return blob.present.highlight_transformed.lines if Feature.enabled?(:jupyter_clean_diffs, @project) + return blob.present.highlight_transformed.lines if Feature.enabled?(:jupyter_clean_diffs, @project, default_enabled: true) blob.present.highlight.lines end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index c24ec347dd6..8d73aa842be 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -108,7 +108,7 @@ module Gitlab end def message_including_template - description = message_including_reply + description = process_message(trim_reply: false, allow_only_quotes: true) template_content = service_desk_setting&.issue_template_content if template_content.present? diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 0f0f4800062..d39fa139abb 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -4,12 +4,13 @@ module Gitlab module Email class ReplyParser - attr_accessor :message + attr_accessor :message, :allow_only_quotes - def initialize(message, trim_reply: true, append_reply: false) + def initialize(message, trim_reply: true, append_reply: false, allow_only_quotes: false) @message = message @trim_reply = trim_reply @append_reply = append_reply + @allow_only_quotes = allow_only_quotes end def execute @@ -25,7 +26,12 @@ module Gitlab # NOTE: We currently don't support empty quotes. # EmailReplyTrimmer allows this as a special case, # so we detect it manually here. - return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } + # + # If allow_only_quotes is true a message where all lines starts with ">" is allowed. + # This could happen if an email has an empty quote, forwarded without any new content. + return "" if body.lines.all? do |l| + l.strip.empty? || (!allow_only_quotes && l.start_with?('>')) + end encoded_body = body.force_encoding(encoding).encode("UTF-8") return encoded_body unless @append_reply diff --git a/spec/controllers/projects/tags_controller_spec.rb b/spec/controllers/projects/tags_controller_spec.rb index d0719643b7f..0045c0a484b 100644 --- a/spec/controllers/projects/tags_controller_spec.rb +++ b/spec/controllers/projects/tags_controller_spec.rb @@ -25,7 +25,7 @@ RSpec.describe Projects::TagsController do with_them do it 'returns 503 status code' do expect_next_instance_of(TagsFinder) do |finder| - expect(finder).to receive(:execute).and_return([[], Gitlab::Git::CommandError.new]) + expect(finder).to receive(:execute).and_raise(Gitlab::Git::CommandError) end get :index, params: { namespace_id: project.namespace.to_param, project_id: project }, format: format diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb index 91ed46a8bb5..dafa639a2d5 100644 --- a/spec/controllers/projects_controller_spec.rb +++ b/spec/controllers/projects_controller_spec.rb @@ -1143,6 +1143,22 @@ RSpec.describe ProjectsController do expect(json_response["Commits"]).to include("123456") end + context 'when gitaly is unavailable' do + before do + expect_next_instance_of(TagsFinder) do |finder| + allow(finder).to receive(:execute).and_raise(Gitlab::Git::CommandError) + end + end + + it 'gets an empty list of tags' do + get :refs, params: { namespace_id: project.namespace, id: project, ref: "123456" } + + expect(json_response["Branches"]).to include("master") + expect(json_response["Tags"]).to eq([]) + expect(json_response["Commits"]).to include("123456") + end + end + context "when preferred language is Japanese" do before do user.update!(preferred_language: 'ja') diff --git a/spec/features/admin/users/users_spec.rb b/spec/features/admin/users/users_spec.rb index 3968b055819..fa943245fcb 100644 --- a/spec/features/admin/users/users_spec.rb +++ b/spec/features/admin/users/users_spec.rb @@ -165,7 +165,7 @@ RSpec.describe 'Admin::Users' do visit admin_users_path - page.within('.filter-two-factor-enabled small') do + page.within('.filter-two-factor-enabled .gl-tab-counter-badge') do expect(page).to have_content('1') end end @@ -182,7 +182,7 @@ RSpec.describe 'Admin::Users' do it 'counts users who have not enabled 2FA' do visit admin_users_path - page.within('.filter-two-factor-disabled small') do + page.within('.filter-two-factor-disabled .gl-tab-counter-badge') do expect(page).to have_content('2') # Including admin end end @@ -201,7 +201,7 @@ RSpec.describe 'Admin::Users' do visit admin_users_path - page.within('.filter-blocked-pending-approval small') do + page.within('.filter-blocked-pending-approval .gl-tab-counter-badge') do expect(page).to have_content('2') end end diff --git a/spec/finders/tags_finder_spec.rb b/spec/finders/tags_finder_spec.rb index fe015d53ac9..790e1cad570 100644 --- a/spec/finders/tags_finder_spec.rb +++ b/spec/finders/tags_finder_spec.rb @@ -8,12 +8,7 @@ RSpec.describe TagsFinder do let_it_be(:repository) { project.repository } def load_tags(params) - tags_finder = described_class.new(repository, params) - tags, error = tags_finder.execute - - expect(error).to eq(nil) - - tags + described_class.new(repository, params).execute end describe '#execute' do @@ -102,14 +97,12 @@ RSpec.describe TagsFinder do end context 'when Gitaly is unavailable' do - it 'returns empty list of tags' do + it 'raises an exception' do expect(Gitlab::GitalyClient).to receive(:call).and_raise(GRPC::Unavailable) tags_finder = described_class.new(repository, {}) - tags, error = tags_finder.execute - expect(error).to be_a(Gitlab::Git::CommandError) - expect(tags).to eq([]) + expect { tags_finder.execute }.to raise_error(Gitlab::Git::CommandError) end end end diff --git a/spec/fixtures/emails/service_desk_all_quoted.eml b/spec/fixtures/emails/service_desk_all_quoted.eml new file mode 100644 index 00000000000..102ebf1f30e --- /dev/null +++ b/spec/fixtures/emails/service_desk_all_quoted.eml @@ -0,0 +1,22 @@ +Return-Path: <jake@adventuretime.ooo> +Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400 +Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400 +Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+email-test-project_id-issue-@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700 +Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700 +Date: Thu, 13 Jun 2013 17:03:48 -0400 +From: Jake the Dog <jake@adventuretime.ooo> +To: incoming+email-test-project_id-issue-@appmail.adventuretime.ooo +Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Subject: The message subject! @all +Mime-Version: 1.0 +Content-Type: text/plain; + charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +X-Sieve: CMU Sieve 2.2 +X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, + 13 Jun 2013 14:03:48 -0700 (PDT) +X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1 + +> This is an empty quote +> someone did forward this email without +> adding any new content. diff --git a/spec/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces_spec.rb b/spec/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces_spec.rb new file mode 100644 index 00000000000..24259b06469 --- /dev/null +++ b/spec/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces_spec.rb @@ -0,0 +1,254 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces, :migration do + include MigrationsHelpers + + context 'when migrating data', :aggregate_failures do + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + + let(:parent_group1) { namespaces.create!(name: 'parent_group1', path: 'parent_group1', visibility_level: 20, type: 'Group') } + let(:parent_group2) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') } + + let(:parent_group1_project) { projects.create!(name: 'parent_group1_project', path: 'parent_group1_project', namespace_id: parent_group1.id, visibility_level: 20) } + let(:parent_group2_project) { projects.create!(name: 'parent_group2_project', path: 'parent_group2_project', namespace_id: parent_group2.id, visibility_level: 20) } + + let(:child_nodes_count) { 2 } + let(:tree_depth) { 3 } + + let(:backfilled_namespace) { nil } + + before do + BackfillProjectNamespaces::TreeGenerator.new(namespaces, projects, [parent_group1, parent_group2], child_nodes_count, tree_depth).build_tree + end + + describe '#up' do + shared_examples 'back-fill project namespaces' do + it 'back-fills all project namespaces' do + start_id = ::Project.minimum(:id) + end_id = ::Project.maximum(:id) + projects_count = ::Project.count + batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil + project_namespaces_count = ::Namespace.where(type: 'Project').count + migration = described_class.new + + expect(projects_count).not_to eq(project_namespaces_count) + expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original + + expect { migration.perform(start_id, end_id, nil, 'up') }.to change(Namespace.where(type: 'Project'), :count) + + expect(projects_count).to eq(::Namespace.where(type: 'Project').count) + check_projects_in_sync_with(Namespace.where(type: 'Project')) + end + + context 'when passing specific group as parameter' do + let(:backfilled_namespace) { parent_group1 } + + it 'back-fills project namespaces for the specified group hierarchy' do + backfilled_namespace_projects = base_ancestor(backfilled_namespace).first.all_projects + start_id = backfilled_namespace_projects.minimum(:id) + end_id = backfilled_namespace_projects.maximum(:id) + group_projects_count = backfilled_namespace_projects.count + batches_count = (group_projects_count / described_class::BATCH_SIZE.to_f).ceil + project_namespaces_in_hierarchy = project_namespaces_in_hierarchy(base_ancestor(backfilled_namespace)) + + migration = described_class.new + + expect(project_namespaces_in_hierarchy.count).to eq(0) + expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original + + expect(group_projects_count).to eq(14) + expect(project_namespaces_in_hierarchy.count).to eq(0) + + migration.perform(start_id, end_id, backfilled_namespace.id, 'up') + + expect(project_namespaces_in_hierarchy.count).to eq(14) + check_projects_in_sync_with(project_namespaces_in_hierarchy) + end + end + + context 'when projects already have project namespaces' do + before do + hierarchy1_projects = base_ancestor(parent_group1).first.all_projects + start_id = hierarchy1_projects.minimum(:id) + end_id = hierarchy1_projects.maximum(:id) + + described_class.new.perform(start_id, end_id, parent_group1.id, 'up') + end + + it 'does not duplicate project namespaces' do + # check there are already some project namespaces but not for all + projects_count = ::Project.count + start_id = ::Project.minimum(:id) + end_id = ::Project.maximum(:id) + batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil + project_namespaces = ::Namespace.where(type: 'Project') + migration = described_class.new + + expect(project_namespaces_in_hierarchy(base_ancestor(parent_group1)).count).to be >= 14 + expect(project_namespaces_in_hierarchy(base_ancestor(parent_group2)).count).to eq(0) + expect(projects_count).not_to eq(project_namespaces.count) + + # run migration again to test we do not generate extra project namespaces + expect(migration).to receive(:batch_insert_namespaces).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_projects).exactly(batches_count).and_call_original + expect(migration).to receive(:batch_update_project_namespaces_traversal_ids).exactly(batches_count).and_call_original + + expect { migration.perform(start_id, end_id, nil, 'up') }.to change(project_namespaces, :count).by(14) + + expect(projects_count).to eq(project_namespaces.count) + end + end + end + + it 'checks no project namespaces exist in the defined hierarchies' do + hierarchy1_project_namespaces = project_namespaces_in_hierarchy(base_ancestor(parent_group1)) + hierarchy2_project_namespaces = project_namespaces_in_hierarchy(base_ancestor(parent_group2)) + hierarchy1_projects_count = base_ancestor(parent_group1).first.all_projects.count + hierarchy2_projects_count = base_ancestor(parent_group2).first.all_projects.count + + expect(hierarchy1_project_namespaces).to be_empty + expect(hierarchy2_project_namespaces).to be_empty + expect(hierarchy1_projects_count).to eq(14) + expect(hierarchy2_projects_count).to eq(14) + end + + context 'back-fill project namespaces in a single batch' do + it_behaves_like 'back-fill project namespaces' + end + + context 'back-fill project namespaces in batches' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + end + + it_behaves_like 'back-fill project namespaces' + end + end + + describe '#down' do + before do + start_id = ::Project.minimum(:id) + end_id = ::Project.maximum(:id) + # back-fill first + described_class.new.perform(start_id, end_id, nil, 'up') + end + + shared_examples 'cleanup project namespaces' do + it 'removes project namespaces' do + projects_count = ::Project.count + start_id = ::Project.minimum(:id) + end_id = ::Project.maximum(:id) + migration = described_class.new + batches_count = (projects_count / described_class::BATCH_SIZE.to_f).ceil + + expect(projects_count).to be > 0 + expect(projects_count).to eq(::Namespace.where(type: 'Project').count) + + expect(migration).to receive(:nullify_project_namespaces_in_projects).exactly(batches_count).and_call_original + expect(migration).to receive(:delete_project_namespace_records).exactly(batches_count).and_call_original + + migration.perform(start_id, end_id, nil, 'down') + + expect(::Project.count).to be > 0 + expect(::Namespace.where(type: 'Project').count).to eq(0) + end + + context 'when passing specific group as parameter' do + let(:backfilled_namespace) { parent_group1 } + + it 'removes project namespaces only for the specific group hierarchy' do + backfilled_namespace_projects = base_ancestor(backfilled_namespace).first.all_projects + start_id = backfilled_namespace_projects.minimum(:id) + end_id = backfilled_namespace_projects.maximum(:id) + group_projects_count = backfilled_namespace_projects.count + batches_count = (group_projects_count / described_class::BATCH_SIZE.to_f).ceil + project_namespaces_in_hierarchy = project_namespaces_in_hierarchy(base_ancestor(backfilled_namespace)) + migration = described_class.new + + expect(project_namespaces_in_hierarchy.count).to eq(14) + expect(migration).to receive(:nullify_project_namespaces_in_projects).exactly(batches_count).and_call_original + expect(migration).to receive(:delete_project_namespace_records).exactly(batches_count).and_call_original + + migration.perform(start_id, end_id, backfilled_namespace.id, 'down') + + expect(::Namespace.where(type: 'Project').count).to be > 0 + expect(project_namespaces_in_hierarchy.count).to eq(0) + end + end + end + + context 'cleanup project namespaces in a single batch' do + it_behaves_like 'cleanup project namespaces' + end + + context 'cleanup project namespaces in batches' do + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + end + + it_behaves_like 'cleanup project namespaces' + end + end + end + + def base_ancestor(ancestor) + ::Namespace.where(id: ancestor.id) + end + + def project_namespaces_in_hierarchy(base_node) + Gitlab::ObjectHierarchy.new(base_node).base_and_descendants.where(type: 'Project') + end + + def check_projects_in_sync_with(namespaces) + project_namespaces_attrs = namespaces.order(:id).pluck(:id, :name, :path, :parent_id, :visibility_level, :shared_runners_enabled) + corresponding_projects_attrs = Project.where(project_namespace_id: project_namespaces_attrs.map(&:first)) + .order(:project_namespace_id).pluck(:project_namespace_id, :name, :path, :namespace_id, :visibility_level, :shared_runners_enabled) + + expect(project_namespaces_attrs).to eq(corresponding_projects_attrs) + end +end + +module BackfillProjectNamespaces + class TreeGenerator + def initialize(namespaces, projects, parent_nodes, child_nodes_count, tree_depth) + parent_nodes_ids = parent_nodes.map(&:id) + + @namespaces = namespaces + @projects = projects + @subgroups_depth = tree_depth + @resource_count = child_nodes_count + @all_groups = [parent_nodes_ids] + end + + def build_tree + (1..@subgroups_depth).each do |level| + parent_level = level - 1 + current_level = level + parent_groups = @all_groups[parent_level] + + parent_groups.each do |parent_id| + @resource_count.times do |i| + group_path = "child#{i}_level#{level}" + project_path = "project#{i}_level#{level}" + sub_group = @namespaces.create!(name: group_path, path: group_path, parent_id: parent_id, visibility_level: 20, type: 'Group') + @projects.create!(name: project_path, path: project_path, namespace_id: sub_group.id, visibility_level: 20) + + track_group_id(current_level, sub_group.id) + end + end + end + end + + def track_group_id(depth_level, group_id) + @all_groups[depth_level] ||= [] + @all_groups[depth_level] << group_id + end + end +end diff --git a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb index 93b3566b709..a3f81bd19d8 100644 --- a/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb +++ b/spec/lib/gitlab/email/handler/service_desk_handler_spec.rb @@ -197,6 +197,17 @@ RSpec.describe Gitlab::Email::Handler::ServiceDeskHandler do end end + context 'when all lines of email are quoted' do + let(:email_raw) { email_fixture('emails/service_desk_all_quoted.eml') } + + it 'creates email with correct body' do + receiver.execute + + issue = Issue.last + expect(issue.description).to include('> This is an empty quote') + end + end + context 'when using custom service desk address' do let(:receiver) { Gitlab::Email::ServiceDeskReceiver.new(email_raw) } diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index 3b01b568fb4..c0d177aff4d 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -21,6 +21,30 @@ RSpec.describe Gitlab::Email::ReplyParser do expect(test_parse_body(fixture_file("emails/no_content_reply.eml"))).to eq("") end + context 'when allow_only_quotes is true' do + it "returns quoted text from email" do + text = test_parse_body(fixture_file("emails/no_content_reply.eml"), allow_only_quotes: true) + + expect(text).to eq( + <<-BODY.strip_heredoc.chomp + > + > + > + > eviltrout posted in 'Adventure Time Sux' on Discourse Meta: + > + > --- + > hey guys everyone knows adventure time sucks! + > + > --- + > Please visit this link to respond: http://localhost:3000/t/adventure-time-sux/1234/3 + > + > To unsubscribe from these emails, visit your [user preferences](http://localhost:3000/user_preferences). + > + BODY + ) + end + end + it "properly renders plaintext-only email" do expect(test_parse_body(fixture_file("emails/plaintext_only.eml"))) .to eq( diff --git a/spec/requests/api/tags_spec.rb b/spec/requests/api/tags_spec.rb index 1aa1ad87be9..cf58809f7cd 100644 --- a/spec/requests/api/tags_spec.rb +++ b/spec/requests/api/tags_spec.rb @@ -208,6 +208,20 @@ RSpec.describe API::Tags do it_behaves_like "get repository tags" end + + context 'when gitaly is unavailable' do + let(:route) { "/projects/#{project_id}/repository/tags" } + + before do + expect_next_instance_of(TagsFinder) do |finder| + allow(finder).to receive(:execute).and_raise(Gitlab::Git::CommandError) + end + end + + it_behaves_like '503 response' do + let(:request) { get api(route, user) } + end + end end describe 'GET /projects/:id/repository/tags/:tag_name' do diff --git a/spec/support/shared_examples/requests/api/status_shared_examples.rb b/spec/support/shared_examples/requests/api/status_shared_examples.rb index 8207190b1dc..b2d50599264 100644 --- a/spec/support/shared_examples/requests/api/status_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/status_shared_examples.rb @@ -76,3 +76,14 @@ RSpec.shared_examples '412 response' do end end end + +RSpec.shared_examples '503 response' do + before do + # Fires the request + request + end + + it 'returns 503' do + expect(response).to have_gitlab_http_status(:service_unavailable) + end +end diff --git a/vendor/project_templates/learn_gitlab_ultimate_trial.tar.gz b/vendor/project_templates/learn_gitlab_ultimate.tar.gz Binary files differindex b8d14bbad93..b8d14bbad93 100644 --- a/vendor/project_templates/learn_gitlab_ultimate_trial.tar.gz +++ b/vendor/project_templates/learn_gitlab_ultimate.tar.gz |
