diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-13 12:11:26 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-13 12:11:26 +0300 |
commit | 12221d835d5f63c4747f0cbd30e4aac8b78e857c (patch) | |
tree | 54a6fdcd715ace3a3cd15c7b89435ca2a742e9b2 | |
parent | e4372ce2ee58813303e4ac906800fbfdd0d5bcf5 (diff) |
Add latest changes from gitlab-org/gitlab@master
50 files changed, 892 insertions, 279 deletions
diff --git a/app/assets/javascripts/reports/components/report_item.vue b/app/assets/javascripts/reports/components/report_item.vue index 8871da8fbd7..918263bfb5c 100644 --- a/app/assets/javascripts/reports/components/report_item.vue +++ b/app/assets/javascripts/reports/components/report_item.vue @@ -53,11 +53,7 @@ export default { }; </script> <template> - <li - :class="{ 'is-dismissed': issue.isDismissed }" - class="report-block-list-issue align-items-center" - data-qa-selector="report_item_row" - > + <li class="report-block-list-issue align-items-center" data-qa-selector="report_item_row"> <component :is="iconComponent" v-if="showReportSectionStatusIcon" diff --git a/app/assets/stylesheets/page_bundles/reports.scss b/app/assets/stylesheets/page_bundles/reports.scss index ce91988cb8a..d0748779f47 100644 --- a/app/assets/stylesheets/page_bundles/reports.scss +++ b/app/assets/stylesheets/page_bundles/reports.scss @@ -49,11 +49,6 @@ display: flex; } -.is-dismissed .report-block-list-issue-description, -.is-dismissed .vulnerability-name-button { - text-decoration: line-through; -} - .report-block-list-issue-description-text::after { content: '\00a0'; } diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 24f86b44841..ab5d248ff8c 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -78,6 +78,30 @@ class BulkImports::Entity < ApplicationRecord ERB::Util.url_encode(source_full_path) end + def pipelines + @pipelines ||= case source_type + when 'group_entity' + BulkImports::Groups::Stage.pipelines + when 'project_entity' + BulkImports::Projects::Stage.pipelines + end + end + + def pipeline_exists?(name) + pipelines.any? { |_, pipeline| pipeline.to_s == name.to_s } + end + + def create_pipeline_trackers! + self.class.transaction do + pipelines.each do |stage, pipeline| + trackers.create!( + stage: stage, + pipeline_name: pipeline + ) + end + end + end + private def validate_parent_is_a_group diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 1b108d5c042..c185470b1c2 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -34,8 +34,8 @@ class BulkImports::Tracker < ApplicationRecord end def pipeline_class - unless BulkImports::Stage.pipeline_exists?(pipeline_name) - raise NameError, "'#{pipeline_name}' is not a valid BulkImport Pipeline" + unless entity.pipeline_exists?(pipeline_name) + raise BulkImports::Error, "'#{pipeline_name}' is not a valid BulkImport Pipeline" end pipeline_name.constantize diff --git a/app/views/shared/deploy_tokens/_form.html.haml b/app/views/shared/deploy_tokens/_form.html.haml index e7bbb351633..652da4b396a 100644 --- a/app/views/shared/deploy_tokens/_form.html.haml +++ b/app/views/shared/deploy_tokens/_form.html.haml @@ -48,7 +48,7 @@ .text-secondary= s_('DeployTokens|Allows read-only access to the package registry.') %fieldset.form-group.form-check - = f.check_box :write_package_registry, class: 'form-check-input' + = f.check_box :write_package_registry, class: 'form-check-input', data: { qa_selector: 'deploy_token_write_package_registry_checkbox' } = f.label :write_package_registry, 'write_package_registry', class: 'label-bold form-check-label' .text-secondary= s_('DeployTokens|Allows read and write access to the package registry.') diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index d6838255916..8bddb6b3143 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2703,7 +2703,7 @@ :worker_name: ServiceDeskEmailReceiverWorker :feature_category: :service_desk :has_external_dependencies: - :urgency: :low + :urgency: :high :resource_boundary: :unknown :weight: 1 :idempotent: diff --git a/app/workers/bulk_import_worker.rb b/app/workers/bulk_import_worker.rb index 61ea0a30be4..fa255d064cc 100644 --- a/app/workers/bulk_import_worker.rb +++ b/app/workers/bulk_import_worker.rb @@ -24,9 +24,9 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker @bulk_import.start! if @bulk_import.created? created_entities.first(next_batch_size).each do |entity| - create_pipeline_tracker_for(entity) + entity.create_pipeline_trackers! - BulkImports::ExportRequestWorker.perform_async(entity.id) + BulkImports::ExportRequestWorker.perform_async(entity.id) if entity.group_entity? BulkImports::EntityWorker.perform_async(entity.id) entity.start! @@ -75,13 +75,4 @@ class BulkImportWorker # rubocop:disable Scalability/IdempotentWorker def re_enqueue BulkImportWorker.perform_in(PERFORM_DELAY, @bulk_import.id) end - - def create_pipeline_tracker_for(entity) - BulkImports::Stage.pipelines.each do |stage, pipeline| - entity.trackers.create!( - stage: stage, - pipeline_name: pipeline - ) - end - end end diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 1514897b2e4..51211834e06 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -11,7 +11,7 @@ class EmailReceiverWorker # rubocop:disable Scalability/IdempotentWorker urgency :high weight 2 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 tags :needs_own_queue attr_accessor :raw diff --git a/app/workers/hashed_storage/migrator_worker.rb b/app/workers/hashed_storage/migrator_worker.rb index c220f663969..03019ae3131 100644 --- a/app/workers/hashed_storage/migrator_worker.rb +++ b/app/workers/hashed_storage/migrator_worker.rb @@ -11,8 +11,7 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # Gitlab::HashedStorage::Migrator#migration_pending? depends on the - # queue size of this worker. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 tags :needs_own_queue # @param [Integer] start initial ID of the batch diff --git a/app/workers/hashed_storage/project_migrate_worker.rb b/app/workers/hashed_storage/project_migrate_worker.rb index 0547e670203..bcc80cc2a70 100644 --- a/app/workers/hashed_storage/project_migrate_worker.rb +++ b/app/workers/hashed_storage/project_migrate_worker.rb @@ -11,8 +11,7 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # Gitlab::HashedStorage::Migrator#migration_pending? depends on the - # queue size of this worker. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 tags :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/project_rollback_worker.rb b/app/workers/hashed_storage/project_rollback_worker.rb index 1a8f5ce926c..07a7ab63718 100644 --- a/app/workers/hashed_storage/project_rollback_worker.rb +++ b/app/workers/hashed_storage/project_rollback_worker.rb @@ -11,8 +11,7 @@ module HashedStorage queue_namespace :hashed_storage loggable_arguments 1 - # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the - # queue size of this worker. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 tags :needs_own_queue attr_reader :project_id diff --git a/app/workers/hashed_storage/rollbacker_worker.rb b/app/workers/hashed_storage/rollbacker_worker.rb index 8302f90fdec..d6a16b4d083 100644 --- a/app/workers/hashed_storage/rollbacker_worker.rb +++ b/app/workers/hashed_storage/rollbacker_worker.rb @@ -11,8 +11,7 @@ module HashedStorage queue_namespace :hashed_storage feature_category :source_code_management - # Gitlab::HashedStorage::Migrator#rollback_pending? depends on the - # queue size of this worker. + # https://gitlab.com/gitlab-org/gitlab/-/issues/340629 tags :needs_own_queue # @param [Integer] start initial ID of the batch diff --git a/app/workers/service_desk_email_receiver_worker.rb b/app/workers/service_desk_email_receiver_worker.rb index f546fce3e8a..c8ab8891856 100644 --- a/app/workers/service_desk_email_receiver_worker.rb +++ b/app/workers/service_desk_email_receiver_worker.rb @@ -6,9 +6,10 @@ class ServiceDeskEmailReceiverWorker < EmailReceiverWorker # rubocop:disable Sca data_consistency :always feature_category :service_desk + urgency :high sidekiq_options retry: 3 - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1087#jobs-written-to-redis-without-passing-through-the-application + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263 tags :needs_own_queue def should_perform? diff --git a/config/feature_flags/development/bulk_import_projects.yml b/config/feature_flags/development/bulk_import_projects.yml new file mode 100644 index 00000000000..853389577cf --- /dev/null +++ b/config/feature_flags/development/bulk_import_projects.yml @@ -0,0 +1,8 @@ +--- +name: bulk_import_projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68873 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339941 +milestone: '14.3' +type: development +group: group::import +default_enabled: false diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md index 84cbfae2230..ed5014b65e1 100644 --- a/doc/administration/operations/cleaning_up_redis_sessions.md +++ b/doc/administration/operations/cleaning_up_redis_sessions.md @@ -1,64 +1,9 @@ --- -stage: Enablement -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2021-12-10' --- -# Cleaning up stale Redis sessions **(FREE SELF)** +This document was moved to [another location](index.md). -Since version 6.2, GitLab stores web user sessions as key-value pairs in Redis. -Prior to GitLab 7.3, user sessions did not automatically expire from Redis. If -you have been running a large GitLab server (thousands of users) since before -GitLab 7.3 we recommend cleaning up stale sessions to compact the Redis -database after you upgrade to GitLab 7.3. You can also perform a cleanup while -still running GitLab 7.2 or older, but in that case new stale sessions will -start building up again after you clean up. - -In GitLab versions prior to 7.3.0, the session keys in Redis are 16-byte -hexadecimal values such as '976aa289e2189b17d7ef525a6702ace9'. Starting with -GitLab 7.3.0, the keys are -prefixed with `session:gitlab:`, so they would look like -`session:gitlab:976aa289e2189b17d7ef525a6702ace9`. Below we describe how to -remove the keys in the old format. - -NOTE: -The instructions below must be modified in accordance with your -configuration settings if you have used the advanced Redis -settings outlined in -[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/README.md). - -First we define a shell function with the proper Redis connection details. - -```shell -rcli() { - # This example works for Omnibus installations of GitLab 7.3 or newer. For an - # installation from source you will have to change the socket path and the - # path to redis-cli. - sudo /opt/gitlab/embedded/bin/redis-cli -s /var/opt/gitlab/redis/redis.socket "$@" -} - -# test the new shell function; the response should be PONG -rcli ping -``` - -Now we do a search to see if there are any session keys in the old format for -us to clean up. - -```shell -# returns the number of old-format session keys in Redis -rcli keys '*' | grep '^[a-f0-9]\{32\}$' | wc -l -``` - -If the number is larger than zero, you can proceed to expire the keys from -Redis. If the number is zero there is nothing to clean up. - -```shell -# Tell Redis to expire each matched key after 600 seconds. -rcli keys '*' | grep '^[a-f0-9]\{32\}$' | awk '{ print "expire", $0, 600 }' | rcli -# This will print '(integer) 1' for each key that gets expired. -``` - -Over the next 15 minutes (10 minutes expiry time plus 5 minutes Redis -background save interval) your Redis database will be compacted. If you are -still using GitLab 7.2, users who are not clicking around in GitLab during the -10 minute expiry window will be signed out of GitLab. +<!-- This redirect file can be deleted after 2021-12-10. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md index 6938f8a7012..68c6e546256 100644 --- a/doc/administration/operations/extra_sidekiq_routing.md +++ b/doc/administration/operations/extra_sidekiq_routing.md @@ -40,6 +40,8 @@ In `/etc/gitlab/gitlab.rb`: ```ruby sidekiq['routing_rules'] = [ + # Do not re-route workers that require their own queue + ['tags=needs_own_queue', nil], # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue ['resource_boundary!=cpu&urgency=high', 'high-urgency'], # Route all database, gitaly and global search workers that are throttled to `throttled` queue @@ -164,3 +166,34 @@ with the migration to avoid losing jobs entirely, especially in a system with long queues of jobs. The migration can be done by following the migration steps mentioned in [Sidekiq job migration](../../raketasks/sidekiq_job_migration.md) + +### Workers that cannot be migrated + +Some workers cannot share a queue with other workers - typically because +they check the size of their own queue - and so must be excluded from +this process. We recommend excluding these from any further worker +routing by adding a rule to keep them in their own queue, for example: + +```ruby +sidekiq['routing_rules'] = [ + ['tags=needs_own_queue', nil], + # ... +] +``` + +These queues will also need to be included in at least one [Sidekiq +queue group](extra_sidekiq_processes.md#start-multiple-processes). + +The following table shows the workers that should have their own queue: + +<!-- markdownlint-disable MD044 --> +| Worker name | Queue name | GitLab issue | +| --- | --- | --- | +| EmailReceiverWorker | `email_receiver` | [gitlab-com/gl-infra/scalability#1263](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) | +| ServiceDeskEmailReceiverWorker | `service_desk_email_receiver` | [gitlab-com/gl-infra/scalability#1263](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) | +| ProjectImportScheduleWorker | `project_import_schedule` | [gitlab-org/gitlab#340630](https://gitlab.com/gitlab-org/gitlab/-/issues/340630) | +| HashedStorage::MigratorWorker | `hashed_storage:hashed_storage_migrator` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | +| HashedStorage::ProjectMigrateWorker | `hashed_storage:hashed_storage_project_migrate` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | +| HashedStorage::ProjectRollbackWorker | `hashed_storage:hashed_storage_project_rollback` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | +| HashedStorage::RollbackerWorker | `hashed_storage:hashed_storage_rollbacker` | [gitlab-org/gitlab#340629](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | +<!-- markdownlint-disable MD044 --> diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 4b16c3b3a7e..7ccfa2739bb 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -8,11 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w Keep your GitLab instance up and running smoothly. -- [Clean up Redis sessions](cleaning_up_redis_sessions.md): Prior to GitLab 7.3, - user sessions did not automatically expire from Redis. If - you have been running a large GitLab server (thousands of users) since before - GitLab 7.3 we recommend cleaning up stale sessions to compact the Redis - database after you upgrade to GitLab 7.3. - [Rake tasks](../../raketasks/index.md): Tasks for common administration and operational processes such as [cleaning up unneeded items from GitLab instance](../../raketasks/cleanup.md), integrity checks, and more. diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 7caac7df64e..b7a651a330c 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -161,10 +161,6 @@ For more information, see [Deployment safety](deployment_safety.md). > - [Feature flag `group_level_protected_environments`](https://gitlab.com/gitlab-org/gitlab/-/issues/331085) removed in GitLab 14.3. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/331085) in GitLab 14.3. -This in-development feature might not be available for your use. There can be -[risks when enabling features still in development](../../administration/feature_flags.md#risks-when-enabling-features-still-in-development). -Refer to this feature's version history for more details. - Typically, large enterprise organizations have an explicit permission boundary between [developers and operators](https://about.gitlab.com/topics/devops/). Developers build and test their code, and operators deploy and monitor the diff --git a/doc/install/aws/eks_clusters_aws.md b/doc/install/aws/eks_clusters_aws.md index 297303e5423..95f9f81f601 100644 --- a/doc/install/aws/eks_clusters_aws.md +++ b/doc/install/aws/eks_clusters_aws.md @@ -5,7 +5,7 @@ group: Alliances info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# EKS cluster provisioning best practices +# EKS cluster provisioning best practices **(FREE SELF)** GitLab can be used to provision an EKS cluster into AWS, however, it necessarily focuses on a basic EKS configuration. Using the AWS tools can help with advanced cluster configuration, automation, and maintenance. diff --git a/doc/install/aws/gitlab_sre_for_aws.md b/doc/install/aws/gitlab_sre_for_aws.md index 8116fed4e39..a2d3a2d0295 100644 --- a/doc/install/aws/gitlab_sre_for_aws.md +++ b/doc/install/aws/gitlab_sre_for_aws.md @@ -7,7 +7,7 @@ description: Doing SRE for GitLab instances and runners on AWS. type: index --- -# GitLab Site Reliability Engineering for AWS +# GitLab Site Reliability Engineering for AWS **(FREE SELF)** ## Known issues list diff --git a/doc/install/docker.md b/doc/install/docker.md index b4faaa684db..9ccdc976488 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# GitLab Docker images +# GitLab Docker images **(FREE SELF)** The GitLab Docker images are monolithic images of GitLab running all the necessary services in a single container. If you instead want to install GitLab diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 23565e6f370..48339144292 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -9,6 +9,11 @@ type: reference, how-to GitLab can integrate with [Kerberos](https://web.mit.edu/kerberos/) as an authentication mechanism. +WARNING: +GitLab CI/CD does not work with a Kerberos-enabled GitLab instance due to an unresolved +[bug in Git CLI](https://lore.kernel.org/git/YKNVop80H8xSTCjz@coredump.intra.peff.net/T/#mab47fd7dcb61fee651f7cc8710b8edc6f62983d5) +that fails to use job token authentication from the GitLab Runners. + ## Overview [Kerberos](https://web.mit.edu/kerberos/) is a secure method for authenticating a request for a service in a diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index 9f79d3cc675..c2d3c540cbf 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -207,10 +207,6 @@ sudo gitlab-rake gitlab:cleanup:sessions:active_sessions_lookup_keys bundle exec rake gitlab:cleanup:sessions:active_sessions_lookup_keys RAILS_ENV=production ``` -## Cleaning up stale Redis sessions - -[Clean up stale sessions](../administration/operations/cleaning_up_redis_sessions.md) to compact the Redis database after you upgrade to GitLab 7.3. - ## Container Registry garbage collection Container Registry can use considerable amounts of disk space. To clear up diff --git a/doc/raketasks/features.md b/doc/raketasks/features.md index 5c2c7b9222f..3248f7370e8 100644 --- a/doc/raketasks/features.md +++ b/doc/raketasks/features.md @@ -10,7 +10,8 @@ This Rake task enables [namespaces](../user/group/index.md#namespaces) for proje ## Enable usernames and namespaces for user projects -This command enables the namespaces feature introduced in GitLab 4.0. It moves every project in its namespace folder. +This command enables the namespaces feature. It moves every project in its +namespace folder. The **repository location changes as part of this task**, so you must **update all your Git URLs** to point to the new location. diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 566270f3774..7dd0e7241eb 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -49,6 +49,15 @@ When GitLab detects a **Denied** license, you can view it in the [license list]( You can view and modify existing policies from the [policies](#policies) tab. ![Edit Policy](img/policies_maintainer_edit_v14_2.png) +## License expressions + +GitLab has limited support for [composite licenses](https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/). +License compliance can read multiple licenses, but always considers them combined using the `AND` operator. For example, +if a dependency has two licenses, and one of them is allowed and the other is denied by the project [policy](#policies), +GitLab evaluates the composite license as _denied_, as this is the safer option. +The ability to support other license expression operators (like `OR`, `WITH`) is tracked +in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/6571). + ## Supported languages and package managers The following languages and package managers are supported. diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 1a709179bf9..aa9221cceee 100644 --- a/lib/bulk_imports/groups/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Pipelines class EntityFinisher def self.ndjson_pipeline? diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb new file mode 100644 index 00000000000..4cec1ad1462 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetProjectsQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!, $cursor: String, $per_page: Int) { + group(fullPath: $full_path) { + projects(includeSubgroups: false, first: $per_page, after: $cursor) { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + name + full_path: fullPath + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data group projects] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb new file mode 100644 index 00000000000..c318675e649 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class ProjectEntitiesPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectsQuery + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + { + source_type: :project_entity, + source_full_path: data['full_path'], + destination_name: data['name'], + destination_namespace: context.entity.group.full_path, + parent_id: context.entity.id + } + end + + def load(context, data) + context.bulk_import.entities.create!(data) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb new file mode 100644 index 00000000000..8c3b6975b73 --- /dev/null +++ b/lib/bulk_imports/groups/stage.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Groups::Pipelines::GroupPipeline, + stage: 0 + }, + avatar: { + pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, + stage: 1 + }, + subgroups: { + pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + stage: 1 + }, + members: { + pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + stage: 1 + }, + labels: { + pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, + stage: 1 + }, + milestones: { + pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + stage: 1 + }, + badges: { + pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + stage: 1 + }, + boards: { + pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, + stage: 2 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 3 + } + }.merge(project_entities_pipeline) + end + + def project_entities_pipeline + if ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + { + project_entities: { + pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline, + stage: 1 + } + } + else + {} + end + end + end + end +end + +::BulkImports::Groups::Stage.prepend_mod_with('BulkImports::Groups::Stage') diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index f27818dae18..6798936576b 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -69,8 +69,8 @@ module BulkImports # Multiple transformers can be defined within a single # pipeline and run sequentially for each record in the # following order: - # - Transformers defined using `transformer` class method # - Instance method `transform` + # - Transformers defined using `transformer` class method # # Instance method `transform` is always the last to run. # diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb new file mode 100644 index 00000000000..2aec496880f --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetProjectQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + description + visibility + archived + created_at: createdAt + shared_runners_enabled: sharedRunnersEnabled + container_registry_enabled: containerRegistryEnabled + only_allow_merge_if_pipeline_succeeds: onlyAllowMergeIfPipelineSucceeds + only_allow_merge_if_all_discussions_are_resolved: onlyAllowMergeIfAllDiscussionsAreResolved + request_access_enabled: requestAccessEnabled + printing_merge_request_link_enabled: printingMergeRequestLinkEnabled + remove_source_branch_after_merge: removeSourceBranchAfterMerge + autoclose_referenced_issues: autocloseReferencedIssues + suggestion_commit_message: suggestionCommitMessage + wiki_enabled: wikiEnabled + } + } + GRAPHQL + end + + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_pipeline.rb new file mode 100644 index 00000000000..c9da33fe8e3 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_pipeline.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectPipeline + include Pipeline + + abort_on_failure! + + extractor ::BulkImports::Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectQuery + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + transformer ::BulkImports::Projects::Transformers::ProjectAttributesTransformer + + def load(context, data) + project = ::Projects::CreateService.new(context.current_user, data).execute + + if project.persisted? + context.entity.update!(project: project) + + project + else + raise(::BulkImports::Error, "Unable to import project #{project.full_path}. #{project.errors.full_messages}.") + end + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb new file mode 100644 index 00000000000..b606003091b --- /dev/null +++ b/lib/bulk_imports/projects/stage.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Projects::Pipelines::ProjectPipeline, + stage: 0 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 1 + } + } + end + end + end +end + +::BulkImports::Projects::Stage.prepend_mod_with('BulkImports::Projects::Stage') diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb new file mode 100644 index 00000000000..24c55d8dbb1 --- /dev/null +++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Transformers + class ProjectAttributesTransformer + PROJECT_IMPORT_TYPE = 'gitlab_project_migration' + + def transform(context, data) + entity = context.entity + visibility = data.delete('visibility') + + data['name'] = entity.destination_name + data['path'] = entity.destination_name.parameterize + data['import_type'] = PROJECT_IMPORT_TYPE + data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] if visibility.present? + data['namespace_id'] = Namespace.find_by_full_path(entity.destination_namespace)&.id if entity.destination_namespace.present? + + data.transform_keys!(&:to_sym) + end + end + end + end +end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index b1bceecbaea..103623cd030 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -2,55 +2,8 @@ module BulkImports class Stage - include Singleton - - CONFIG = { - group: { - pipeline: BulkImports::Groups::Pipelines::GroupPipeline, - stage: 0 - }, - avatar: { - pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, - stage: 1 - }, - subgroups: { - pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, - stage: 1 - }, - members: { - pipeline: BulkImports::Groups::Pipelines::MembersPipeline, - stage: 1 - }, - labels: { - pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, - stage: 1 - }, - milestones: { - pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, - stage: 1 - }, - badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, - stage: 1 - }, - boards: { - pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, - stage: 2 - }, - finisher: { - pipeline: BulkImports::Groups::Pipelines::EntityFinisher, - stage: 3 - } - }.freeze - def self.pipelines - instance.pipelines - end - - def self.pipeline_exists?(name) - pipelines.any? do |(_, pipeline)| - pipeline.to_s == name.to_s - end + new.pipelines end def pipelines @@ -65,9 +18,8 @@ module BulkImports private def config - @config ||= CONFIG + # To be implemented in a sub-class + NotImplementedError end end end - -::BulkImports::Stage.prepend_mod_with('BulkImports::Stage') diff --git a/qa/qa/page/project/settings/deploy_tokens.rb b/qa/qa/page/project/settings/deploy_tokens.rb index db1f6f68ec6..7b61c81154a 100644 --- a/qa/qa/page/project/settings/deploy_tokens.rb +++ b/qa/qa/page/project/settings/deploy_tokens.rb @@ -10,6 +10,7 @@ module QA element :deploy_token_expires_at_field element :deploy_token_read_repository_checkbox element :deploy_token_read_package_registry_checkbox + element :deploy_token_write_package_registry_checkbox element :deploy_token_read_registry_checkbox element :create_deploy_token_button end @@ -28,9 +29,10 @@ module QA fill_element(:deploy_token_expires_at_field, expires_at.to_s + "\n") end - def fill_scopes(read_repository: false, read_registry: false, read_package_registry: false) + def fill_scopes(read_repository: false, read_registry: false, read_package_registry: false, write_package_registry: false) check_element(:deploy_token_read_repository_checkbox) if read_repository check_element(:deploy_token_read_package_registry_checkbox) if read_package_registry + check_element(:deploy_token_write_package_registry_checkbox) if write_package_registry check_element(:deploy_token_read_registry_checkbox) if read_registry end diff --git a/qa/qa/resource/deploy_token.rb b/qa/qa/resource/deploy_token.rb index cd638ad2f85..151454c37b1 100644 --- a/qa/qa/resource/deploy_token.rb +++ b/qa/qa/resource/deploy_token.rb @@ -37,7 +37,7 @@ module QA setting.expand_deploy_tokens do |page| page.fill_token_name(name) page.fill_token_expires_at(expires_at) - page.fill_scopes(read_repository: true, read_package_registry: true) + page.fill_scopes(read_repository: true, read_package_registry: true, write_package_registry: true) page.add_token end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb index 40f67b6b646..81fb187df85 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb @@ -1,15 +1,16 @@ # frozen_string_literal: true module QA - RSpec.describe 'Create', quarantine: { only: { subdomain: :staging }, issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/323990', type: :flaky } do + RSpec.describe 'Create' do describe 'Merge request rebasing' do - it 'user rebases source branch of merge request', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1596' do + let(:merge_request) { Resource::MergeRequest.fabricate_via_api! } + + before do Flow::Login.sign_in + end - project = Resource::Project.fabricate_via_api! do |project| - project.name = "only-fast-forward" - end - project.visit! + it 'user rebases source branch of merge request', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1596' do + merge_request.project.visit! Page::Project::Menu.perform(&:go_to_general_settings) Page::Project::Settings::Main.perform do |main| @@ -18,13 +19,8 @@ module QA end end - merge_request = Resource::MergeRequest.fabricate! do |merge_request| - merge_request.project = project - merge_request.title = 'Needs rebasing' - end - Resource::Repository::ProjectPush.fabricate! do |push| - push.project = project + push.project = merge_request.project push.file_name = "other.txt" push.file_content = "New file added!" push.new_branch = false @@ -33,7 +29,7 @@ module QA merge_request.visit! Page::MergeRequest::Show.perform do |merge_request| - expect(merge_request).to have_content('Needs rebasing') + expect(merge_request).to have_content('Merge blocked: the source branch must be rebased onto the target branch.') expect(merge_request).to be_fast_forward_not_possible expect(merge_request).not_to have_merge_button diff --git a/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb index 52ba6da1cae..5a3b4388f0c 100644 --- a/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/npm_registry_spec.rb @@ -3,10 +3,11 @@ module QA RSpec.describe 'Package', :orchestrated, :packages, :reliable, :object_storage do describe 'npm registry' do + using RSpec::Parameterized::TableSyntax include Runtime::Fixtures let!(:registry_scope) { Runtime::Namespace.sandbox_name } - let(:auth_token) do + let!(:personal_access_token) do unless Page::Main::Menu.perform(&:signed_in?) Flow::Login.sign_in end @@ -14,6 +15,13 @@ module QA Resource::PersonalAccessToken.fabricate!.token end + let(:project_deploy_token) do + Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + deploy_token.name = 'npm-deploy-token' + deploy_token.project = project + end + end + let(:uri) { URI.parse(Runtime::Scenario.gitlab_address) } let(:gitlab_address_with_port) { "#{uri.scheme}://#{uri.host}:#{uri.port}" } let(:gitlab_host_with_port) { "#{uri.host}:#{uri.port}" } @@ -109,16 +117,6 @@ module QA } end - let(:npmrc) do - { - file_path: '.npmrc', - content: <<~NPMRC - //#{gitlab_host_with_port}/api/v4/projects/#{project.id}/packages/npm/:_authToken=#{auth_token} - @#{registry_scope}:registry=#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/npm/ - NPMRC - } - end - let(:package) do Resource::Package.init do |package| package.name = "@#{registry_scope}/#{project.name}" @@ -133,72 +131,101 @@ module QA another_project.remove_via_api! end - it 'push and pull a npm package via CI', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1811' do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_deploy_yaml, - npmrc, - package_json - ]) - end - - project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('deploy') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = another_project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([ - gitlab_ci_install_yaml - ]) - end - - another_project.visit! - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('install') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - job.click_browse_button - end - - Page::Project::Artifact::Show.perform do |artifacts| - artifacts.go_to_directory('node_modules') - artifacts.go_to_directory("@#{registry_scope}") - expect(artifacts).to have_content( "#{project.name}") - end - - project.visit! - Page::Project::Menu.perform(&:click_packages_link) - - Page::Project::Packages::Index.perform do |index| - expect(index).to have_package(package.name) + where(:authentication_token_type, :token_name) do + :personal_access_token | 'Personal Access Token' + :ci_job_token | 'CI Job Token' + :project_deploy_token | 'Deploy Token' + end - index.click_package(package.name) + with_them do + let(:auth_token) do + case authentication_token_type + when :personal_access_token + "\"#{personal_access_token}\"" + when :ci_job_token + '${CI_JOB_TOKEN}' + when :project_deploy_token + "\"#{project_deploy_token.password}\"" + end end - Page::Project::Packages::Show.perform do |show| - expect(show).to have_package_info(package.name, "1.0.0") - - show.click_delete + let(:npmrc) do + { + file_path: '.npmrc', + content: <<~NPMRC + //#{gitlab_host_with_port}/api/v4/projects/#{project.id}/packages/npm/:_authToken=#{auth_token} + @#{registry_scope}:registry=#{gitlab_address_with_port}/api/v4/projects/#{project.id}/packages/npm/ + NPMRC + } end - Page::Project::Packages::Index.perform do |index| - expect(index).to have_content("Package deleted successfully") - expect(index).not_to have_package(package.name) + it "push and pull a npm package via CI using a #{params[:token_name]}", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1772' do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + gitlab_ci_deploy_yaml, + npmrc, + package_json + ]) + end + + project.visit! + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('deploy') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = another_project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([ + gitlab_ci_install_yaml + ]) + end + + another_project.visit! + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('install') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + job.click_browse_button + end + + Page::Project::Artifact::Show.perform do |artifacts| + artifacts.go_to_directory('node_modules') + artifacts.go_to_directory("@#{registry_scope}") + expect(artifacts).to have_content( "#{project.name}") + end + + project.visit! + Page::Project::Menu.perform(&:click_packages_link) + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_package(package.name) + + index.click_package(package.name) + end + + Page::Project::Packages::Show.perform do |show| + expect(show).to have_package_info(package.name, "1.0.0") + + show.click_delete + end + + Page::Project::Packages::Index.perform do |index| + expect(index).to have_content("Package deleted successfully") + expect(index).not_to have_package(package.name) + end end end end diff --git a/spec/lib/bulk_imports/groups/pipelines/entity_finisher_spec.rb b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb index b97aeb435b9..c1a9ea7b7e2 100644 --- a/spec/lib/bulk_imports/groups/pipelines/entity_finisher_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe BulkImports::Groups::Pipelines::EntityFinisher do +RSpec.describe BulkImports::Common::Pipelines::EntityFinisher do it 'updates the entity status to finished' do entity = create(:bulk_import_entity, :started) pipeline_tracker = create(:bulk_import_tracker, entity: entity) diff --git a/spec/lib/bulk_imports/groups/graphql/get_projects_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_projects_query_spec.rb new file mode 100644 index 00000000000..1a7c5a4993c --- /dev/null +++ b/spec/lib/bulk_imports/groups/graphql/get_projects_query_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Graphql::GetProjectsQuery do + describe '#variables' do + it 'returns valid variables based on entity information' do + tracker = create(:bulk_import_tracker) + context = BulkImports::Pipeline::Context.new(tracker) + + query = GraphQL::Query.new( + GitlabSchema, + described_class.to_s, + variables: described_class.variables(context) + ) + result = GitlabSchema.static_validator.validate(query) + + expect(result[:errors]).to be_empty + end + + context 'with invalid variables' do + it 'raises an error' do + expect { GraphQL::Query.new(GitlabSchema, described_class.to_s, variables: 'invalid') }.to raise_error(ArgumentError) + end + end + end + + describe '#data_path' do + it 'returns data path' do + expected = %w[data group projects nodes] + + expect(described_class.data_path).to eq(expected) + end + end + + describe '#page_info_path' do + it 'returns pagination information path' do + expected = %w[data group projects page_info] + + expect(described_class.page_info_path).to eq(expected) + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb new file mode 100644 index 00000000000..5b6c93e695f --- /dev/null +++ b/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Pipelines::ProjectEntitiesPipeline do + let_it_be(:user) { create(:user) } + let_it_be(:destination_group) { create(:group) } + + let_it_be(:entity) do + create( + :bulk_import_entity, + group: destination_group, + destination_namespace: destination_group.full_path + ) + end + + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } + + let(:extracted_data) do + BulkImports::Pipeline::ExtractedData.new(data: { + 'name' => 'project', + 'full_path' => 'group/project' + }) + end + + subject { described_class.new(context) } + + describe '#run' do + before do + allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| + allow(extractor).to receive(:extract).and_return(extracted_data) + end + + destination_group.add_owner(user) + end + + it 'creates project entity' do + expect { subject.run }.to change(BulkImports::Entity, :count).by(1) + + project_entity = BulkImports::Entity.last + + expect(project_entity.source_type).to eq('project_entity') + expect(project_entity.source_full_path).to eq('group/project') + expect(project_entity.destination_name).to eq('project') + expect(project_entity.destination_namespace).to eq(destination_group.full_path) + end + end + + describe 'pipeline parts' do + it { expect(described_class).to include_module(BulkImports::Pipeline) } + it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } + + it 'has extractors' do + expect(described_class.get_extractor).to eq( + klass: BulkImports::Common::Extractors::GraphqlExtractor, + options: { + query: BulkImports::Groups::Graphql::GetProjectsQuery + } + ) + end + + it 'has transformers' do + expect(described_class.transformers).to contain_exactly( + { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } + ) + end + end +end diff --git a/spec/lib/bulk_imports/stage_spec.rb b/spec/lib/bulk_imports/groups/stage_spec.rb index 4398b00e7e9..81c0ffc14d4 100644 --- a/spec/lib/bulk_imports/stage_spec.rb +++ b/spec/lib/bulk_imports/groups/stage_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'fast_spec_helper' +require 'spec_helper' -RSpec.describe BulkImports::Stage do +RSpec.describe BulkImports::Groups::Stage do let(:pipelines) do [ [0, BulkImports::Groups::Pipelines::GroupPipeline], @@ -19,18 +19,21 @@ RSpec.describe BulkImports::Stage do describe '.pipelines' do it 'list all the pipelines with their stage number, ordered by stage' do expect(described_class.pipelines & pipelines).to eq(pipelines) - expect(described_class.pipelines.last.last).to eq(BulkImports::Groups::Pipelines::EntityFinisher) + expect(described_class.pipelines.last.last).to eq(BulkImports::Common::Pipelines::EntityFinisher) end - end - describe '.pipeline_exists?' do - it 'returns true when the given pipeline name exists in the pipelines list' do - expect(described_class.pipeline_exists?(BulkImports::Groups::Pipelines::GroupPipeline)).to eq(true) - expect(described_class.pipeline_exists?('BulkImports::Groups::Pipelines::GroupPipeline')).to eq(true) + it 'includes project entities pipeline' do + stub_feature_flags(bulk_import_projects: true) + + expect(described_class.pipelines).to include([1, BulkImports::Groups::Pipelines::ProjectEntitiesPipeline]) end - it 'returns false when the given pipeline name exists in the pipelines list' do - expect(described_class.pipeline_exists?('BulkImports::Groups::Pipelines::InexistentPipeline')).to eq(false) + context 'when bulk_import_projects feature flag is disabled' do + it 'does not include project entities pipeline' do + stub_feature_flags(bulk_import_projects: false) + + expect(described_class.pipelines.flatten).not_to include(BulkImports::Groups::Pipelines::ProjectEntitiesPipeline) + end end end end diff --git a/spec/lib/bulk_imports/projects/pipelines/project_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/project_pipeline_spec.rb new file mode 100644 index 00000000000..c53c0849931 --- /dev/null +++ b/spec/lib/bulk_imports/projects/pipelines/project_pipeline_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Projects::Pipelines::ProjectPipeline do + describe '#run' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let_it_be(:entity) do + create( + :bulk_import_entity, + source_type: :project_entity, + bulk_import: bulk_import, + source_full_path: 'source/full/path', + destination_name: 'My Destination Project', + destination_namespace: group.full_path + ) + end + + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } + + let(:project_data) do + { + 'visibility' => 'private', + 'created_at' => 10.days.ago, + 'archived' => false, + 'shared_runners_enabled' => true, + 'container_registry_enabled' => true, + 'only_allow_merge_if_pipeline_succeeds' => true, + 'only_allow_merge_if_all_discussions_are_resolved' => true, + 'request_access_enabled' => true, + 'printing_merge_request_link_enabled' => true, + 'remove_source_branch_after_merge' => true, + 'autoclose_referenced_issues' => true, + 'suggestion_commit_message' => 'message', + 'wiki_enabled' => true + } + end + + subject(:project_pipeline) { described_class.new(context) } + + before do + allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| + allow(extractor).to receive(:extract).and_return(BulkImports::Pipeline::ExtractedData.new(data: project_data)) + end + + group.add_owner(user) + end + + it 'imports new project into destination group', :aggregate_failures do + expect { project_pipeline.run }.to change { Project.count }.by(1) + + project_path = 'my-destination-project' + imported_project = Project.find_by_path(project_path) + + expect(imported_project).not_to be_nil + expect(imported_project.group).to eq(group) + expect(imported_project.suggestion_commit_message).to eq('message') + expect(imported_project.archived?).to eq(project_data['archived']) + expect(imported_project.shared_runners_enabled?).to eq(project_data['shared_runners_enabled']) + expect(imported_project.container_registry_enabled?).to eq(project_data['container_registry_enabled']) + expect(imported_project.only_allow_merge_if_pipeline_succeeds?).to eq(project_data['only_allow_merge_if_pipeline_succeeds']) + expect(imported_project.only_allow_merge_if_all_discussions_are_resolved?).to eq(project_data['only_allow_merge_if_all_discussions_are_resolved']) + expect(imported_project.request_access_enabled?).to eq(project_data['request_access_enabled']) + expect(imported_project.printing_merge_request_link_enabled?).to eq(project_data['printing_merge_request_link_enabled']) + expect(imported_project.remove_source_branch_after_merge?).to eq(project_data['remove_source_branch_after_merge']) + expect(imported_project.autoclose_referenced_issues?).to eq(project_data['autoclose_referenced_issues']) + expect(imported_project.wiki_enabled?).to eq(project_data['wiki_enabled']) + end + end + + describe 'pipeline parts' do + it { expect(described_class).to include_module(BulkImports::Pipeline) } + it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } + + it 'has extractors' do + expect(described_class.get_extractor) + .to eq( + klass: BulkImports::Common::Extractors::GraphqlExtractor, + options: { query: BulkImports::Projects::Graphql::GetProjectQuery } + ) + end + + it 'has transformers' do + expect(described_class.transformers) + .to contain_exactly( + { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil }, + { klass: BulkImports::Projects::Transformers::ProjectAttributesTransformer, options: nil } + ) + end + end +end diff --git a/spec/lib/bulk_imports/projects/stage_spec.rb b/spec/lib/bulk_imports/projects/stage_spec.rb new file mode 100644 index 00000000000..428812a34ef --- /dev/null +++ b/spec/lib/bulk_imports/projects/stage_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Projects::Stage do + let(:pipelines) do + [ + [0, BulkImports::Projects::Pipelines::ProjectPipeline], + [1, BulkImports::Common::Pipelines::EntityFinisher] + ] + end + + describe '.pipelines' do + it 'list all the pipelines with their stage number, ordered by stage' do + expect(described_class.pipelines).to eq(pipelines) + end + end +end diff --git a/spec/lib/bulk_imports/projects/transformers/project_attributes_transformer_spec.rb b/spec/lib/bulk_imports/projects/transformers/project_attributes_transformer_spec.rb new file mode 100644 index 00000000000..822bb9a5605 --- /dev/null +++ b/spec/lib/bulk_imports/projects/transformers/project_attributes_transformer_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Projects::Transformers::ProjectAttributesTransformer do + describe '#transform' do + let_it_be(:user) { create(:user) } + let_it_be(:destination_group) { create(:group) } + let_it_be(:project) { create(:project, name: 'My Source Project') } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let_it_be(:entity) do + create( + :bulk_import_entity, + source_type: :project_entity, + bulk_import: bulk_import, + source_full_path: 'source/full/path', + destination_name: 'Destination Project Name', + destination_namespace: destination_group.full_path + ) + end + + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } + + let(:data) do + { + 'name' => 'source_name', + 'visibility' => 'private' + } + end + + subject(:transformed_data) { described_class.new.transform(context, data) } + + it 'transforms name to destination name' do + expect(transformed_data[:name]).to eq(entity.destination_name) + end + + it 'adds path as parameterized name' do + expect(transformed_data[:path]).to eq(entity.destination_name.parameterize) + end + + it 'transforms visibility level' do + visibility = data['visibility'] + + expect(transformed_data).not_to have_key(:visibility) + expect(transformed_data[:visibility_level]).to eq(Gitlab::VisibilityLevel.string_options[visibility]) + end + + it 'adds import type' do + expect(transformed_data[:import_type]).to eq(described_class::PROJECT_IMPORT_TYPE) + end + + describe 'namespace_id' do + context 'when destination namespace is present' do + it 'adds namespace_id' do + expect(transformed_data[:namespace_id]).to eq(destination_group.id) + end + end + + context 'when destination namespace is blank' do + it 'does not add namespace_id key' do + entity = create( + :bulk_import_entity, + source_type: :project_entity, + bulk_import: bulk_import, + source_full_path: 'source/full/path', + destination_name: 'Destination Project Name', + destination_namespace: '' + ) + + context = double(entity: entity) + + expect(described_class.new.transform(context, data)).not_to have_key(:namespace_id) + end + end + end + + it 'converts all keys to symbols' do + expect(transformed_data.keys).to contain_exactly(:name, :path, :import_type, :visibility_level, :namespace_id) + end + end +end diff --git a/spec/models/bulk_imports/entity_spec.rb b/spec/models/bulk_imports/entity_spec.rb index 11a3e53dd16..c1cbe61885f 100644 --- a/spec/models/bulk_imports/entity_spec.rb +++ b/spec/models/bulk_imports/entity_spec.rb @@ -154,4 +154,57 @@ RSpec.describe BulkImports::Entity, type: :model do expect(described_class.all_human_statuses).to contain_exactly('created', 'started', 'finished', 'failed') end end + + describe '#pipelines' do + context 'when entity is group' do + it 'returns group pipelines' do + entity = build(:bulk_import_entity, :group_entity) + + expect(entity.pipelines.flatten).to include(BulkImports::Groups::Pipelines::GroupPipeline) + end + end + + context 'when entity is project' do + it 'returns project pipelines' do + entity = build(:bulk_import_entity, :project_entity) + + expect(entity.pipelines.flatten).to include(BulkImports::Projects::Pipelines::ProjectPipeline) + end + end + end + + describe '#create_pipeline_trackers!' do + context 'when entity is group' do + it 'creates trackers for group entity' do + entity = create(:bulk_import_entity, :group_entity) + entity.create_pipeline_trackers! + + expect(entity.trackers.count).to eq(BulkImports::Groups::Stage.pipelines.count) + expect(entity.trackers.map(&:pipeline_name)).to include(BulkImports::Groups::Pipelines::GroupPipeline.to_s) + end + end + + context 'when entity is project' do + it 'creates trackers for project entity' do + entity = create(:bulk_import_entity, :project_entity) + entity.create_pipeline_trackers! + + expect(entity.trackers.count).to eq(BulkImports::Projects::Stage.pipelines.count) + expect(entity.trackers.map(&:pipeline_name)).to include(BulkImports::Projects::Pipelines::ProjectPipeline.to_s) + end + end + end + + describe '#pipeline_exists?' do + let_it_be(:entity) { create(:bulk_import_entity, :group_entity) } + + it 'returns true when the given pipeline name exists in the pipelines list' do + expect(entity.pipeline_exists?(BulkImports::Groups::Pipelines::GroupPipeline)).to eq(true) + expect(entity.pipeline_exists?('BulkImports::Groups::Pipelines::GroupPipeline')).to eq(true) + end + + it 'returns false when the given pipeline name exists in the pipelines list' do + expect(entity.pipeline_exists?('BulkImports::Groups::Pipelines::InexistentPipeline')).to eq(false) + end + end end diff --git a/spec/models/bulk_imports/tracker_spec.rb b/spec/models/bulk_imports/tracker_spec.rb index 0f00aeb9c1d..7f0a7d4f1ae 100644 --- a/spec/models/bulk_imports/tracker_spec.rb +++ b/spec/models/bulk_imports/tracker_spec.rb @@ -66,7 +66,7 @@ RSpec.describe BulkImports::Tracker, type: :model do describe '#pipeline_class' do it 'returns the pipeline class' do - pipeline_class = BulkImports::Stage.pipelines.first[1] + pipeline_class = BulkImports::Groups::Stage.pipelines.first[1] tracker = create(:bulk_import_tracker, pipeline_name: pipeline_class) expect(tracker.pipeline_class).to eq(pipeline_class) @@ -77,7 +77,7 @@ RSpec.describe BulkImports::Tracker, type: :model do expect { tracker.pipeline_class } .to raise_error( - NameError, + BulkImports::Error, "'InexistingPipeline' is not a valid BulkImport Pipeline" ) end diff --git a/spec/workers/bulk_import_worker_spec.rb b/spec/workers/bulk_import_worker_spec.rb index 205bf23f36d..b67c5c62f76 100644 --- a/spec/workers/bulk_import_worker_spec.rb +++ b/spec/workers/bulk_import_worker_spec.rb @@ -84,17 +84,20 @@ RSpec.describe BulkImportWorker do expect { subject.perform(bulk_import.id) } .to change(BulkImports::Tracker, :count) - .by(BulkImports::Stage.pipelines.size * 2) + .by(BulkImports::Groups::Stage.pipelines.size * 2) expect(entity_1.trackers).not_to be_empty expect(entity_2.trackers).not_to be_empty end context 'when there are created entities to process' do - it 'marks a batch of entities as started, enqueues EntityWorker, ExportRequestWorker and reenqueues' do + let_it_be(:bulk_import) { create(:bulk_import, :created) } + + before do stub_const("#{described_class}::DEFAULT_BATCH_SIZE", 1) + end - bulk_import = create(:bulk_import, :created) + it 'marks a batch of entities as started, enqueues EntityWorker, ExportRequestWorker and reenqueues' do create(:bulk_import_entity, :created, bulk_import: bulk_import) create(:bulk_import_entity, :created, bulk_import: bulk_import) @@ -106,6 +109,16 @@ RSpec.describe BulkImportWorker do expect(bulk_import.entities.map(&:status_name)).to contain_exactly(:created, :started) end + + context 'when there are project entities to process' do + it 'does not enqueue ExportRequestWorker' do + create(:bulk_import_entity, :created, :project_entity, bulk_import: bulk_import) + + expect(BulkImports::ExportRequestWorker).not_to receive(:perform_async) + + subject.perform(bulk_import.id) + end + end end context 'when exception occurs' do diff --git a/spec/workers/bulk_imports/pipeline_worker_spec.rb b/spec/workers/bulk_imports/pipeline_worker_spec.rb index 972a4158194..56f28654ac5 100644 --- a/spec/workers/bulk_imports/pipeline_worker_spec.rb +++ b/spec/workers/bulk_imports/pipeline_worker_spec.rb @@ -21,6 +21,10 @@ RSpec.describe BulkImports::PipelineWorker do before do stub_const('FakePipeline', pipeline_class) + + allow(BulkImports::Groups::Stage) + .to receive(:pipelines) + .and_return([[0, pipeline_class]]) end it 'runs the given pipeline successfully' do @@ -30,12 +34,6 @@ RSpec.describe BulkImports::PipelineWorker do pipeline_name: 'FakePipeline' ) - expect(BulkImports::Stage) - .to receive(:pipeline_exists?) - .with('FakePipeline') - .twice - .and_return(true) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger) .to receive(:info) @@ -110,7 +108,7 @@ RSpec.describe BulkImports::PipelineWorker do expect(Gitlab::ErrorTracking) .to receive(:track_exception) .with( - instance_of(NameError), + instance_of(BulkImports::Error), entity_id: entity.id, pipeline_name: pipeline_tracker.pipeline_name ) @@ -157,10 +155,10 @@ RSpec.describe BulkImports::PipelineWorker do before do stub_const('NdjsonPipeline', ndjson_pipeline) - allow(BulkImports::Stage) - .to receive(:pipeline_exists?) - .with('NdjsonPipeline') - .and_return(true) + + allow(BulkImports::Groups::Stage) + .to receive(:pipelines) + .and_return([[0, ndjson_pipeline]]) end it 'runs the pipeline successfully' do |