diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-01 18:07:40 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-01 18:07:40 +0300 |
commit | 04e74bf311de04c1334343a35fe9954953c6413d (patch) | |
tree | d2d3e62216291c93bb24a79d9221b0eca3efadcb | |
parent | 098ec8c914f61780b33bb18e929e25ef59dfb175 (diff) |
Add latest changes from gitlab-org/gitlab@master
58 files changed, 793 insertions, 280 deletions
diff --git a/app/finders/projects_finder.rb b/app/finders/projects_finder.rb index 582075efc4e..dca3d12f3c9 100644 --- a/app/finders/projects_finder.rb +++ b/app/finders/projects_finder.rb @@ -55,7 +55,7 @@ class ProjectsFinder < UnionFinder collection = Project.wrap_with_cte(collection) if use_cte collection = filter_projects(collection) - if params[:sort] == 'similarity' && params[:search] && Feature.enabled?(:project_finder_similarity_sort, current_user) + if params[:sort] == 'similarity' && params[:search] collection.sorted_by_similarity_desc(params[:search]) else sort(collection) diff --git a/app/finders/security/jobs_finder.rb b/app/finders/security/jobs_finder.rb index b8649520c20..cffc1d52a3e 100644 --- a/app/finders/security/jobs_finder.rb +++ b/app/finders/security/jobs_finder.rb @@ -38,7 +38,7 @@ module Security def execute return [] if @job_types.empty? - if Feature.enabled?(:ci_build_metadata_config) + if Feature.enabled?(:ci_build_metadata_config, pipeline.project, default_enabled: :yaml) find_jobs else find_jobs_legacy diff --git a/app/graphql/mutations/ci/job_token_scope/add_project.rb b/app/graphql/mutations/ci/job_token_scope/add_project.rb new file mode 100644 index 00000000000..30f98a537b5 --- /dev/null +++ b/app/graphql/mutations/ci/job_token_scope/add_project.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Mutations + module Ci + module JobTokenScope + class AddProject < BaseMutation + include FindsProject + + graphql_name 'CiJobTokenScopeAddProject' + + authorize :admin_project + + argument :project_path, GraphQL::ID_TYPE, + required: true, + description: 'The project that the CI job token scope belongs to.' + + argument :target_project_path, GraphQL::ID_TYPE, + required: true, + description: 'The project to be added to the CI job token scope.' + + field :ci_job_token_scope, + Types::Ci::JobTokenScopeType, + null: true, + description: "The CI job token's scope of access." + + def resolve(project_path:, target_project_path:) + project = authorized_find!(project_path) + target_project = Project.find_by_full_path(target_project_path) + + result = ::Ci::JobTokenScope::AddProjectService + .new(project, current_user) + .execute(target_project) + + if result.success? + { + ci_job_token_scope: ::Ci::JobToken::Scope.new(project), + errors: [] + } + else + { + ci_job_token_scope: nil, + errors: [result.message] + } + end + end + end + end + end +end diff --git a/app/graphql/types/mutation_type.rb b/app/graphql/types/mutation_type.rb index d321efecb1b..7c4d5c06628 100644 --- a/app/graphql/types/mutation_type.rb +++ b/app/graphql/types/mutation_type.rb @@ -99,6 +99,7 @@ module Types mount_mutation Mutations::Ci::CiCdSettingsUpdate mount_mutation Mutations::Ci::Job::Play mount_mutation Mutations::Ci::Job::Retry + mount_mutation Mutations::Ci::JobTokenScope::AddProject mount_mutation Mutations::Ci::Runner::Update, feature_flag: :runner_graphql_query mount_mutation Mutations::Ci::Runner::Delete, feature_flag: :runner_graphql_query mount_mutation Mutations::Ci::RunnersRegistrationToken::Reset, feature_flag: :runner_graphql_query diff --git a/app/helpers/integrations_helper.rb b/app/helpers/integrations_helper.rb index 59fba0058d7..f37f7518dec 100644 --- a/app/helpers/integrations_helper.rb +++ b/app/helpers/integrations_helper.rb @@ -172,10 +172,6 @@ module IntegrationsHelper name: integration.to_param } end - - def show_service_templates_nav_link? - Feature.disabled?(:disable_service_templates, type: :development, default_enabled: :yaml) - end end IntegrationsHelper.prepend_mod_with('IntegrationsHelper') diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index bb2dac5cd43..7a3779abf5c 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -22,8 +22,8 @@ module Ci validates :build, presence: true validates :secrets, json_schema: { filename: 'build_metadata_secrets' } - serialize :config_options, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize - serialize :config_variables, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize + serialize :config_variables, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize chronic_duration_attr_reader :timeout_human_readable, :timeout diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 601637ea32a..114435d5a21 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -77,7 +77,7 @@ module Ci def write_metadata_attribute(legacy_key, metadata_key, value) # save to metadata or this model depending on the state of feature flag - if Feature.enabled?(:ci_build_metadata_config) + if Feature.enabled?(:ci_build_metadata_config, project, default_enabled: :yaml) ensure_metadata.write_attribute(metadata_key, value) write_attribute(legacy_key, nil) else diff --git a/app/services/ci/job_token_scope/add_project_service.rb b/app/services/ci/job_token_scope/add_project_service.rb new file mode 100644 index 00000000000..6ae0a8428c2 --- /dev/null +++ b/app/services/ci/job_token_scope/add_project_service.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Ci + module JobTokenScope + class AddProjectService < ::BaseService + TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND = "The target_project that you are attempting to access does " \ + "not exist or you don't have permission to perform this action" + + def execute(target_project) + if error_response = validation_error(target_project) + return error_response + end + + link = add_project!(target_project) + ServiceResponse.success(payload: { project_link: link }) + + rescue ActiveRecord::RecordNotUnique + ServiceResponse.error(message: "Target project is already in the job token scope") + rescue ActiveRecord::RecordInvalid => e + ServiceResponse.error(message: e.message) + end + + private + + def validation_error(target_project) + unless project.ci_job_token_scope_enabled? + return ServiceResponse.error(message: "Job token scope is disabled for this project") + end + + unless can?(current_user, :admin_project, project) + return ServiceResponse.error(message: "Insufficient permissions to modify the job token scope") + end + + unless target_project + return ServiceResponse.error(message: TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND) + end + + unless can?(current_user, :read_project, target_project) + return ServiceResponse.error(message: TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND) + end + + nil + end + + def add_project!(target_project) + ::Ci::JobToken::ProjectScopeLink.create!( + source_project: project, + target_project: target_project, + added_by: current_user + ) + end + end + end +end diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb index 2039ece8281..c941734ac40 100644 --- a/app/services/ci/queue/builds_table_strategy.rb +++ b/app/services/ci/queue/builds_table_strategy.rb @@ -18,7 +18,7 @@ module Ci .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id') .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') - if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_queueing_disaster_recovery_disable_fair_scheduling, runner, type: :ops, default_enabled: :yaml) # if disaster recovery is enabled, we fallback to FIFO scheduling relation.order('ci_builds.id ASC') else diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb index b49c6871d00..55d5cb96a0a 100644 --- a/app/services/ci/queue/pending_builds_strategy.rb +++ b/app/services/ci/queue/pending_builds_strategy.rb @@ -18,7 +18,7 @@ module Ci .joins('LEFT JOIN project_features ON ci_pending_builds.project_id = project_features.project_id') .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0') - if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_queueing_disaster_recovery_disable_fair_scheduling, runner, type: :ops, default_enabled: :yaml) # if disaster recovery is enabled, we fallback to FIFO scheduling relation.order('ci_pending_builds.build_id ASC') else diff --git a/app/views/layouts/nav/sidebar/_admin.html.haml b/app/views/layouts/nav/sidebar/_admin.html.haml index 21c3d7cb7e2..5c8465c32ff 100644 --- a/app/views/layouts/nav/sidebar/_admin.html.haml +++ b/app/views/layouts/nav/sidebar/_admin.html.haml @@ -209,19 +209,6 @@ = render_if_exists 'layouts/nav/sidebar/credentials_link' - - if show_service_templates_nav_link? - = nav_link(controller: :services) do - = link_to admin_application_settings_services_path do - .nav-icon-container - = sprite_icon('template') - %span.nav-item-name - = _('Service Templates') - %ul.sidebar-sub-level-items.is-fly-out-only - = nav_link(controller: :services, html_options: { class: "fly-out-top-item" } ) do - = link_to admin_application_settings_services_path do - %strong.fly-out-top-item-name - = _('Service Templates') - = nav_link(controller: :labels) do = link_to admin_labels_path do .nav-icon-container diff --git a/config/feature_flags/development/block_external_fork_network_mirrors.yml b/config/feature_flags/development/block_external_fork_network_mirrors.yml index 81ff34a3d6e..8c313bc9273 100644 --- a/config/feature_flags/development/block_external_fork_network_mirrors.yml +++ b/config/feature_flags/development/block_external_fork_network_mirrors.yml @@ -5,4 +5,4 @@ rollout_issue_url: milestone: '14.0' type: development group: group::source code -default_enabled: false +default_enabled: true diff --git a/config/feature_flags/development/ci_build_metadata_config.yml b/config/feature_flags/development/ci_build_metadata_config.yml index 2caf8be7187..774b3f8fdc7 100644 --- a/config/feature_flags/development/ci_build_metadata_config.yml +++ b/config/feature_flags/development/ci_build_metadata_config.yml @@ -1,7 +1,7 @@ --- name: ci_build_metadata_config introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7238 -rollout_issue_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/330954 milestone: '11.7' type: development group: group::pipeline execution diff --git a/config/feature_flags/development/erase_traces_from_already_archived_jobs_when_archiving_again.yml b/config/feature_flags/development/erase_traces_from_already_archived_jobs_when_archiving_again.yml deleted file mode 100644 index a4e18f99349..00000000000 --- a/config/feature_flags/development/erase_traces_from_already_archived_jobs_when_archiving_again.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: erase_traces_from_already_archived_jobs_when_archiving_again -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56353 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326679 -milestone: "13.11" -type: development -group: group::pipeline execution -default_enabled: true diff --git a/config/feature_flags/development/disable_service_templates.yml b/config/feature_flags/development/jira_issue_details_edit_status.yml index 5e9972a2171..9d64707a79f 100644 --- a/config/feature_flags/development/disable_service_templates.yml +++ b/config/feature_flags/development/jira_issue_details_edit_status.yml @@ -1,8 +1,8 @@ --- -name: disable_service_templates -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59098 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/327436 -milestone: '13.12' +name: jira_issue_details_edit_status +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60092 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/330628 +milestone: '14.1' type: development group: group::ecosystem -default_enabled: true +default_enabled: false diff --git a/config/feature_flags/development/project_finder_similarity_sort.yml b/config/feature_flags/development/project_finder_similarity_sort.yml deleted file mode 100644 index 4dadee97fd5..00000000000 --- a/config/feature_flags/development/project_finder_similarity_sort.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: project_finder_similarity_sort -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43136 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/263249 -milestone: '13.5' -type: development -group: group::threat insights -default_enabled: false diff --git a/config/feature_flags/development/use_traversal_ids_for_ancestors.yml b/config/feature_flags/development/use_traversal_ids_for_ancestors.yml index 57804957192..ae5f154a6bc 100644 --- a/config/feature_flags/development/use_traversal_ids_for_ancestors.yml +++ b/config/feature_flags/development/use_traversal_ids_for_ancestors.yml @@ -1,7 +1,7 @@ --- name: use_traversal_ids_for_ancestors introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57137 -rollout_issue_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/334952 milestone: '13.12' type: development group: group::access diff --git a/config/feature_flags/ops/ci_queueing_disaster_recovery_disable_fair_scheduling.yml b/config/feature_flags/ops/ci_queueing_disaster_recovery_disable_fair_scheduling.yml new file mode 100644 index 00000000000..65c1e0cfb53 --- /dev/null +++ b/config/feature_flags/ops/ci_queueing_disaster_recovery_disable_fair_scheduling.yml @@ -0,0 +1,8 @@ +--- +name: ci_queueing_disaster_recovery_disable_fair_scheduling +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56658 +rollout_issue_url: +milestone: "13.12" +type: ops +group: group::pipeline execution +default_enabled: false diff --git a/config/feature_flags/ops/ci_queueing_disaster_recovery.yml b/config/feature_flags/ops/ci_queueing_disaster_recovery_disable_quota.yml index 6a31c29ec1d..ebb55796aaa 100644 --- a/config/feature_flags/ops/ci_queueing_disaster_recovery.yml +++ b/config/feature_flags/ops/ci_queueing_disaster_recovery_disable_quota.yml @@ -1,5 +1,5 @@ --- -name: ci_queueing_disaster_recovery +name: ci_queueing_disaster_recovery_disable_quota introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56658 rollout_issue_url: milestone: "13.12" diff --git a/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb new file mode 100644 index 00000000000..cbd0d0ea3a2 --- /dev/null +++ b/db/post_migrate/20210622141148_schedule_delete_orphaned_deployments.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class ScheduleDeleteOrphanedDeployments < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'DeleteOrphanedDeployments' + BATCH_SIZE = 100_000 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + define_batchable_model('deployments'), + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb b/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb new file mode 100644 index 00000000000..371298aef62 --- /dev/null +++ b/db/post_migrate/20210629101712_remove_deprecated_modsecurity_columns.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveDeprecatedModsecurityColumns < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_clusters_applications_ingress_on_modsecurity' + + def up + remove_column :clusters_applications_ingress, :modsecurity_enabled if column_exists?(:clusters_applications_ingress, :modsecurity_enabled) + remove_column :clusters_applications_ingress, :modsecurity_mode if column_exists?(:clusters_applications_ingress, :modsecurity_mode) + end + + def down + add_column :clusters_applications_ingress, :modsecurity_enabled, :boolean unless column_exists?(:clusters_applications_ingress, :modsecurity_enabled) + add_column :clusters_applications_ingress, :modsecurity_mode, :smallint, null: false, default: 0 unless column_exists?(:clusters_applications_ingress, :modsecurity_mode) + + add_concurrent_index :clusters_applications_ingress, [:modsecurity_enabled, :modsecurity_mode, :cluster_id], name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20210622141148 b/db/schema_migrations/20210622141148 new file mode 100644 index 00000000000..ba693e271aa --- /dev/null +++ b/db/schema_migrations/20210622141148 @@ -0,0 +1 @@ +432954295d6f3a2a45f3deef42b547ffe42501beaea4f376e1be51cf148de671
\ No newline at end of file diff --git a/db/schema_migrations/20210629101712 b/db/schema_migrations/20210629101712 new file mode 100644 index 00000000000..4b4a88bdfa3 --- /dev/null +++ b/db/schema_migrations/20210629101712 @@ -0,0 +1 @@ +28e448810fdf8bab4de44d45acac862e752f578b5b8fd77b885a385b9ef16b2d
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7a274ebfdf8..1d46fc11129 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11682,9 +11682,7 @@ CREATE TABLE clusters_applications_ingress ( cluster_ip character varying, status_reason text, external_ip character varying, - external_hostname character varying, - modsecurity_enabled boolean, - modsecurity_mode smallint DEFAULT 0 NOT NULL + external_hostname character varying ); CREATE SEQUENCE clusters_applications_ingress_id_seq @@ -23169,8 +23167,6 @@ CREATE UNIQUE INDEX index_clusters_applications_helm_on_cluster_id ON clusters_a CREATE UNIQUE INDEX index_clusters_applications_ingress_on_cluster_id ON clusters_applications_ingress USING btree (cluster_id); -CREATE INDEX index_clusters_applications_ingress_on_modsecurity ON clusters_applications_ingress USING btree (modsecurity_enabled, modsecurity_mode, cluster_id); - CREATE UNIQUE INDEX index_clusters_applications_jupyter_on_cluster_id ON clusters_applications_jupyter USING btree (cluster_id); CREATE INDEX index_clusters_applications_jupyter_on_oauth_application_id ON clusters_applications_jupyter USING btree (oauth_application_id); diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 7fab424ac93..3cfbc8ae74b 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -162,6 +162,7 @@ The following user actions are recorded: - Failed second-factor authentication attempt ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in GitLab 13.5) - A user's personal access token was successfully created or revoked ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6) - A failed attempt to create or revoke a user's personal access token ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6) +- Administrator added or removed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323905) in GitLab 14.1) Instance events can also be accessed via the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). diff --git a/doc/administration/troubleshooting/defcon.md b/doc/administration/troubleshooting/defcon.md index 7cae6ea1c8f..1b263f70b46 100644 --- a/doc/administration/troubleshooting/defcon.md +++ b/doc/administration/troubleshooting/defcon.md @@ -10,7 +10,7 @@ type: reference This document describes a feature that allows you to disable some important but computationally expensive parts of the application to relieve stress on the database during an ongoing downtime. -## `ci_queueing_disaster_recovery` +## `ci_queueing_disaster_recovery_disable_fair_scheduling` This feature flag, if temporarily enabled, disables fair scheduling on shared runners. This can help to reduce system resource usage on the `jobs/request` endpoint @@ -20,6 +20,16 @@ Side effects: - In case of a large backlog of jobs, the jobs are processed in the order they were put in the system, instead of balancing the jobs across many projects. + +## `ci_queueing_disaster_recovery_disable_quota` + +This feature flag, if temporarily enabled, disables enforcing CI minutes quota +on shared runners. This can help to reduce system resource usage on the +`jobs/request` endpoint by significantly reducing the computations being +performed. + +Side effects: + - Projects which are out of quota will be run. This affects only jobs created during the last hour, as prior jobs are canceled by a periodic background worker (`StuckCiJobsWorker`). diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 4a9c75967a3..5e62a75ab49 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -780,6 +780,26 @@ Input type: `CiCdSettingsUpdateInput` | <a id="mutationcicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationcicdsettingsupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.ciJobTokenScopeAddProject` + +Input type: `CiJobTokenScopeAddProjectInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationcijobtokenscopeaddprojectclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationcijobtokenscopeaddprojectprojectpath"></a>`projectPath` | [`ID!`](#id) | The project that the CI job token scope belongs to. | +| <a id="mutationcijobtokenscopeaddprojecttargetprojectpath"></a>`targetProjectPath` | [`ID!`](#id) | The project to be added to the CI job token scope. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationcijobtokenscopeaddprojectcijobtokenscope"></a>`ciJobTokenScope` | [`CiJobTokenScopeType`](#cijobtokenscopetype) | The CI job token's scope of access. | +| <a id="mutationcijobtokenscopeaddprojectclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationcijobtokenscopeaddprojecterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.clusterAgentDelete` Input type: `ClusterAgentDeleteInput` diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 72ef5b85e27..83f02717de3 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -139,18 +139,8 @@ always take the latest Secret Detection artifact available. ### Enable Secret Detection via an automatic merge request **(ULTIMATE SELF)** -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4496) in GitLab 13.11. -> - [Deployed behind a feature flag](../../../user/feature_flags.md), enabled by default. -> - Enabled on GitLab.com. -> - Recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-configure-secret-detection-via-a-merge-request). **(ULTIMATE SELF)** - -WARNING: -This feature might not be available to you. Check the **version history** note above for details. - -There can be -[risks when disabling released features](../../../user/feature_flags.md#risks-when-disabling-released-features). -Refer to this feature's version history for more details. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4496) in GitLab 13.11, behind a feature flag, enabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/329886) in GitLab 14.1. To enable Secret Detection in a project, you can create a merge request from the Security Configuration page. @@ -409,22 +399,3 @@ secret_detection: variables: GIT_DEPTH: 100 ``` - -### Enable or disable Configure Secret Detection via a Merge Request - -Configure Secret Detection via a Merge Request is under development but ready for production use. -It is deployed behind a feature flag that is **enabled by default**. -[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to disable it. - -To enable it: - -```ruby -Feature.enable(:sec_secret_detection_ui_enable) -``` - -To disable it: - -```ruby -Feature.disable(:sec_secret_detection_ui_enable) -``` diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 182551c554e..921c40eef73 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -48,6 +48,10 @@ Iteration cadences automate some common iteration tasks. They can be used to automatically create iterations every 1, 2, 3, 4, or 6 weeks. They can also be configured to automatically roll over incomplete issues to the next iteration. +With iteration cadences enabled, you must first +[create an iteration cadence](#create-an-iteration-cadence) before you can +[create an iteration](#create-an-iteration). + ### Create an iteration cadence Prerequisites: @@ -94,7 +98,7 @@ To create an iteration: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Issues > Iterations**. -1. Select the three-dot menu (**{ellipsis_v}**) > **Add iteration** for the cadence you want to add to. +1. Select **New iteration**. 1. Enter the title, a description (optional), a start date, and a due date. 1. Select **Create iteration**. The iteration details page opens. @@ -191,13 +195,13 @@ can enable it. To enable it: ```ruby -Feature.enable(:iterations_cadences) +Feature.enable(:iteration_cadences) ``` To disable it: ```ruby -Feature.disable(:iterations_cadences) +Feature.disable(:iteration_cadences) ``` <!-- ## Troubleshooting diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb new file mode 100644 index 00000000000..9ac4111ff0f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned deployments. + class DeleteOrphanedDeployments + include Database::MigrationHelpers + + def perform(start_id, end_id) + orphaned_deployments + .where(id: start_id..end_id) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + + def orphaned_deployments + define_batchable_model('deployments') + .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 84eb860a168..f9798023838 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -189,7 +189,7 @@ module Gitlab raise ArchiveError, 'Job is not finished yet' unless job.complete? if trace_artifact - unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + unsafe_trace_cleanup! raise AlreadyArchivedError, 'Could not archive again' end diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 6afc21be4e0..df6d3eb7d0a 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -27,21 +27,18 @@ module Gitlab # connect with Spamcheck @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') - creds = + @creds = if Rails.env.development? || Rails.env.test? :this_channel_is_insecure else GRPC::Core::ChannelCredentials.new end - - @stub = ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, creds, - timeout: DEFAULT_TIMEOUT_SECS) end def issue_spam?(spam_issue:, user:, context: {}) issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) - response = @stub.check_for_spam_issue(issue, + response = grpc_client.check_for_spam_issue(issue, metadata: { 'authorization' => Gitlab::CurrentSettings.spam_check_api_key }) verdict = convert_verdict_to_gitlab_constant(response.verdict) @@ -100,6 +97,16 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: ar_timestamp.to_time.to_i, nanos: ar_timestamp.to_time.nsec) end + + def grpc_client + @grpc_client ||= ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, @creds, + interceptors: interceptors, + timeout: DEFAULT_TIMEOUT_SECS) + end + + def interceptors + [Labkit::Correlation::GRPC::ClientInterceptor.instance] + end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 77e0e2ca96c..0b1acaf7dd8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -169,6 +169,16 @@ module Gitlab end end + def deep_symbolized_access(data) + if data.is_a?(Array) + data.map(&method(:deep_symbolized_access)) + elsif data.is_a?(Hash) + data.deep_symbolize_keys + else + data + end + end + def string_to_ip_object(str) return unless str diff --git a/lib/serializers/symbolized_json.rb b/lib/serializers/symbolized_json.rb new file mode 100644 index 00000000000..78192ce3132 --- /dev/null +++ b/lib/serializers/symbolized_json.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Serializers + # Make the resulting hash have deep symbolized keys + class SymbolizedJson + class << self + def dump(obj) + obj + end + + def load(data) + return if data.nil? + + Gitlab::Utils.deep_symbolized_access(data) + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 78bd3883442..b81208c4324 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -18403,9 +18403,15 @@ msgstr "" msgid "JiraService|Events for %{noteable_model_name} are disabled." msgstr "" +msgid "JiraService|Failed to load Jira issue statuses. View the issue in Jira, or reload the page." +msgstr "" + msgid "JiraService|Failed to load Jira issue. View the issue in Jira, or reload the page." msgstr "" +msgid "JiraService|Failed to update Jira issue status. View the issue in Jira, or reload the page." +msgstr "" + msgid "JiraService|Fetch issue types for this Jira project" msgstr "" @@ -18454,6 +18460,9 @@ msgstr "" msgid "JiraService|Move to Done" msgstr "" +msgid "JiraService|No available statuses" +msgstr "" + msgid "JiraService|Not all data may be displayed here. To view more details or make changes to this issue, go to %{linkStart}Jira%{linkEnd}." msgstr "" @@ -28829,6 +28838,12 @@ msgstr "" msgid "SecurityPolicies|Latest scan" msgstr "" +msgid "SecurityPolicies|Network" +msgstr "" + +msgid "SecurityPolicies|Policy type" +msgstr "" + msgid "SecurityPolicies|Scan execution" msgstr "" diff --git a/spec/deprecation_toolkit_env.rb b/spec/deprecation_toolkit_env.rb index 00d66ff3cdc..811c932b613 100644 --- a/spec/deprecation_toolkit_env.rb +++ b/spec/deprecation_toolkit_env.rb @@ -60,6 +60,7 @@ module DeprecationToolkitEnv activerecord-6.0.3.7/lib/active_record/relation.rb asciidoctor-2.0.12/lib/asciidoctor/extensions.rb attr_encrypted-3.1.0/lib/attr_encrypted/adapters/active_record.rb + gitlab-labkit-0.18.0/lib/labkit/correlation/grpc/client_interceptor.rb ] end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index 965bda8b4d6..e2ebd8b267d 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -309,65 +309,6 @@ RSpec.describe 'Admin updates settings' do end end - context 'when Service Templates are enabled' do - before do - stub_feature_flags(disable_service_templates: false) - visit general_admin_application_settings_path - end - - it 'shows Service Templates link' do - expect(page).to have_link('Service Templates') - end - - context 'when the Slack Notifications Service template is active' do - before do - create(:service, :template, type: 'SlackService', active: true) - - visit general_admin_application_settings_path - end - - it 'change Slack Notifications Service template settings', :js do - first(:link, 'Service Templates').click - click_link 'Slack notifications' - fill_in 'Webhook', with: 'http://localhost' - fill_in 'Username', with: 'test_user' - fill_in 'service[push_channel]', with: '#test_channel' - page.check('Notify only broken pipelines') - page.select 'All branches', from: 'Branches to be notified' - page.select 'Match any of the labels', from: 'Labels to be notified behavior' - - check_all_events - click_button 'Save changes' - - expect(page).to have_content 'Application settings saved successfully' - - click_link 'Slack notifications' - - expect(page.all('input[type=checkbox]')).to all(be_checked) - expect(find_field('Webhook').value).to eq 'http://localhost' - expect(find_field('Username').value).to eq 'test_user' - expect(find('[name="service[push_channel]"]').value).to eq '#test_channel' - end - - it 'defaults Deployment events to false for chat notification template settings', :js do - first(:link, 'Service Templates').click - click_link 'Slack notifications' - - expect(find_field('Deployment')).not_to be_checked - end - end - end - - context 'When Service templates are disabled' do - before do - stub_feature_flags(disable_service_templates: true) - end - - it 'does not show Service Templates link' do - expect(page).not_to have_link('Service Templates') - end - end - context 'Integration page', :js do before do visit integrations_admin_application_settings_path diff --git a/spec/finders/projects_finder_spec.rb b/spec/finders/projects_finder_spec.rb index b8b5e2c3bb7..21b5b2f6130 100644 --- a/spec/finders/projects_finder_spec.rb +++ b/spec/finders/projects_finder_spec.rb @@ -31,10 +31,6 @@ RSpec.describe ProjectsFinder do let(:use_cte) { true } let(:finder) { described_class.new(params: params.merge(use_cte: use_cte), current_user: current_user, project_ids_relation: project_ids_relation) } - before do - stub_feature_flags(project_finder_similarity_sort: false) - end - subject { finder.execute } shared_examples 'ProjectFinder#execute examples' do @@ -368,32 +364,28 @@ RSpec.describe ProjectsFinder do end describe 'sorting' do + let_it_be(:more_projects) do + [ + create(:project, :internal, group: group, name: 'projA', path: 'projA'), + create(:project, :internal, group: group, name: 'projABC', path: 'projABC'), + create(:project, :internal, group: group, name: 'projAB', path: 'projAB') + ] + end + context 'when sorting by a field' do let(:params) { { sort: 'name_asc' } } - it { is_expected.to eq([internal_project, public_project]) } + it { is_expected.to eq(([internal_project, public_project] + more_projects).sort_by { |p| p[:name] }) } end context 'when sorting by similarity' do let(:params) { { sort: 'similarity', search: 'pro' } } - let_it_be(:internal_project2) do - create(:project, :internal, group: group, name: 'projA', path: 'projA') - end - - let_it_be(:internal_project3) do - create(:project, :internal, group: group, name: 'projABC', path: 'projABC') - end - - let_it_be(:internal_project4) do - create(:project, :internal, group: group, name: 'projAB', path: 'projAB') - end - - before do - stub_feature_flags(project_finder_similarity_sort: current_user) - end + it { is_expected.to eq([more_projects[0], more_projects[2], more_projects[1]]) } + end - it { is_expected.to eq([internal_project2, internal_project4, internal_project3]) } + context 'when no sort is provided' do + it { is_expected.to eq(([internal_project, public_project] + more_projects).sort_by { |p| p[:id] }.reverse) } end end diff --git a/spec/fixtures/api/schemas/cluster_status.json b/spec/fixtures/api/schemas/cluster_status.json index ce62655648b..6f9535286ed 100644 --- a/spec/fixtures/api/schemas/cluster_status.json +++ b/spec/fixtures/api/schemas/cluster_status.json @@ -37,8 +37,6 @@ "hostname": { "type": ["string", "null"] }, "email": { "type": ["string", "null"] }, "stack": { "type": ["string", "null"] }, - "modsecurity_enabled": { "type": ["boolean", "null"] }, - "modsecurity_mode": {"type": ["integer", "0"]}, "host": {"type": ["string", "null"]}, "port": {"type": ["integer", "514"]}, "protocol": {"type": ["integer", "0"]}, diff --git a/spec/graphql/mutations/ci/job_token_scope/add_project_spec.rb b/spec/graphql/mutations/ci/job_token_scope/add_project_spec.rb new file mode 100644 index 00000000000..6416ec20dcb --- /dev/null +++ b/spec/graphql/mutations/ci/job_token_scope/add_project_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Mutations::Ci::JobTokenScope::AddProject do + let(:mutation) do + described_class.new(object: nil, context: { current_user: current_user }, field: nil) + end + + describe '#resolve' do + let_it_be(:project) { create(:project) } + let_it_be(:target_project) { create(:project) } + + let(:target_project_path) { target_project.full_path } + + subject do + mutation.resolve(project_path: project.full_path, target_project_path: target_project_path) + end + + context 'when user is not logged in' do + let(:current_user) { nil } + + it 'raises error' do + expect { subject }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable) + end + end + + context 'when user is logged in' do + let(:current_user) { create(:user) } + + context 'when user does not have permissions to admin project' do + it 'raises error' do + expect { subject }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable) + end + end + + context 'when user has permissions to admin project and read target project' do + before do + project.add_maintainer(current_user) + target_project.add_guest(current_user) + end + + it 'adds target project to the job token scope' do + expect do + expect(subject).to include(ci_job_token_scope: be_present, errors: be_empty) + end.to change { Ci::JobToken::ProjectScopeLink.count }.by(1) + end + + context 'when the service returns an error' do + let(:service) { double(:service) } + + it 'returns an error response' do + expect(::Ci::JobTokenScope::AddProjectService).to receive(:new).with(project, current_user).and_return(service) + expect(service).to receive(:execute).with(target_project).and_return(ServiceResponse.error(message: 'The error message')) + + expect(subject.fetch(:ci_job_token_scope)).to be_nil + expect(subject.fetch(:errors)).to include("The error message") + end + end + end + end + end +end diff --git a/spec/graphql/resolvers/projects_resolver_spec.rb b/spec/graphql/resolvers/projects_resolver_spec.rb index 2f2aacb9ad5..2685115d1a2 100644 --- a/spec/graphql/resolvers/projects_resolver_spec.rb +++ b/spec/graphql/resolvers/projects_resolver_spec.rb @@ -27,10 +27,6 @@ RSpec.describe Resolvers::ProjectsResolver do private_group.add_developer(user) end - before do - stub_feature_flags(project_finder_similarity_sort: false) - end - context 'when user is not logged in' do let(:current_user) { nil } @@ -83,6 +79,7 @@ RSpec.describe Resolvers::ProjectsResolver do context 'when user is logged in' do let(:current_user) { user } + let(:visible_projecs) { [project, other_project, group_project, private_project, private_group_project] } context 'when no filters are applied' do it 'returns all visible projects for the user' do @@ -129,21 +126,24 @@ RSpec.describe Resolvers::ProjectsResolver do end end - context 'when sort is similarity' do + context 'when sorting' do let_it_be(:named_project1) { create(:project, :public, name: 'projAB', path: 'projAB') } let_it_be(:named_project2) { create(:project, :public, name: 'projABC', path: 'projABC') } let_it_be(:named_project3) { create(:project, :public, name: 'projA', path: 'projA') } + let_it_be(:named_projects) { [named_project1, named_project2, named_project3] } - let(:filters) { { search: 'projA', sort: 'similarity' } } - - it 'returns projects in order of similarity to search' do - stub_feature_flags(project_finder_similarity_sort: current_user) + context 'when sorting by similarity' do + let(:filters) { { search: 'projA', sort: 'similarity' } } - is_expected.to eq([named_project3, named_project1, named_project2]) + it 'returns projects in order of similarity to search' do + is_expected.to eq([named_project3, named_project1, named_project2]) + end end - it 'returns projects in any order if flag is off' do - is_expected.to match_array([named_project3, named_project1, named_project2]) + context 'when no sort is provided' do + it 'returns projects in descending order by id' do + is_expected.to match_array((visible_projecs + named_projects).sort_by { |p| p[:id]}.reverse ) + end end end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb new file mode 100644 index 00000000000..c4039b85459 --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_deployments_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedDeployments, :migration, schema: 20210617161348 do + let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let!(:project) { table(:projects).create!(namespace_id: namespace.id) } + let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } + let(:background_migration_jobs) { table(:background_migration_jobs) } + + before do + create_deployment!(environment.id, project.id) + create_deployment!(non_existing_record_id, project.id) + end + + it 'deletes only orphaned deployments' do + expect(valid_deployments.pluck(:id)).not_to be_empty + expect(orphaned_deployments.pluck(:id)).not_to be_empty + + subject.perform(table(:deployments).minimum(:id), table(:deployments).maximum(:id)) + + expect(valid_deployments.pluck(:id)).not_to be_empty + expect(orphaned_deployments.pluck(:id)).to be_empty + end + + it 'marks jobs as done' do + first_job = background_migration_jobs.create!( + class_name: 'DeleteOrphanedDeployments', + arguments: [table(:deployments).minimum(:id), table(:deployments).minimum(:id)] + ) + + second_job = background_migration_jobs.create!( + class_name: 'DeleteOrphanedDeployments', + arguments: [table(:deployments).maximum(:id), table(:deployments).maximum(:id)] + ) + + subject.perform(table(:deployments).minimum(:id), table(:deployments).minimum(:id)) + + expect(first_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:succeeded]) + expect(second_job.reload.status).to eq(Gitlab::Database::BackgroundMigrationJob.statuses[:pending]) + end + + private + + def valid_deployments + table(:deployments).where('EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') + end + + def orphaned_deployments + table(:deployments).where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') + end + + def create_deployment!(environment_id, project_id) + table(:deployments).create!( + environment_id: environment_id, + project_id: project_id, + ref: 'master', + tag: false, + sha: 'x', + status: 1, + iid: table(:deployments).count + 1) + end +end diff --git a/spec/lib/gitlab/spamcheck/client_spec.rb b/spec/lib/gitlab/spamcheck/client_spec.rb index 491e5e9a662..15e963fe423 100644 --- a/spec/lib/gitlab/spamcheck/client_spec.rb +++ b/spec/lib/gitlab/spamcheck/client_spec.rb @@ -7,7 +7,7 @@ RSpec.describe Gitlab::Spamcheck::Client do let(:endpoint) { 'grpc://grpc.test.url' } let_it_be(:user) { create(:user, organization: 'GitLab') } - let(:verdict_value) { nil } + let(:verdict_value) { ::Spamcheck::SpamVerdict::Verdict::ALLOW } let(:error_value) { "" } let(:attribs_value) do @@ -56,6 +56,13 @@ RSpec.describe Gitlab::Spamcheck::Client do expect(subject).to eq([expected, { "monitorMode" => "false" }, ""]) end end + + it 'includes interceptors' do + expect_next_instance_of(::Gitlab::Spamcheck::Client) do |client| + expect(client).to receive(:interceptors).and_call_original + end + subject + end end describe "#build_issue_protobuf", :aggregate_failures do diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index a7ccce0aaab..f1601294c07 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -351,6 +351,22 @@ RSpec.describe Gitlab::Utils do end end + describe '.deep_symbolized_access' do + let(:hash) do + { "variables" => [{ "key" => "VAR1", "value" => "VALUE2" }] } + end + + subject { described_class.deep_symbolized_access(hash) } + + it 'allows to access hash keys with symbols' do + expect(subject[:variables]).to be_a(Array) + end + + it 'allows to access array keys with symbols' do + expect(subject[:variables].first[:key]).to eq('VAR1') + end + end + describe '.try_megabytes_to_bytes' do context 'when the size can be converted to megabytes' do it 'returns the size in megabytes' do diff --git a/spec/lib/serializers/symbolized_json_spec.rb b/spec/lib/serializers/symbolized_json_spec.rb new file mode 100644 index 00000000000..b30fb074ddd --- /dev/null +++ b/spec/lib/serializers/symbolized_json_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Serializers::SymbolizedJson do + describe '.dump' do + let(:obj) { { key: "value" } } + + subject { described_class.dump(obj) } + + it 'returns a hash' do + is_expected.to eq(obj) + end + end + + describe '.load' do + let(:data_string) { '{"key":"value","variables":[{"key":"VAR1","value":"VALUE1"}]}' } + let(:data_hash) { Gitlab::Json.parse(data_string) } + + context 'when loading a hash' do + subject { described_class.load(data_hash) } + + it 'decodes a string' do + is_expected.to be_a(Hash) + end + + it 'allows to access with symbols' do + expect(subject[:key]).to eq('value') + expect(subject[:variables].first[:key]).to eq('VAR1') + end + end + + context 'when loading a nil' do + subject { described_class.load(nil) } + + it 'returns nil' do + is_expected.to be_nil + end + end + end +end diff --git a/spec/migrations/schedule_delete_orphaned_deployments_spec.rb b/spec/migrations/schedule_delete_orphaned_deployments_spec.rb new file mode 100644 index 00000000000..618958a3d90 --- /dev/null +++ b/spec/migrations/schedule_delete_orphaned_deployments_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe ScheduleDeleteOrphanedDeployments, :sidekiq, schema: 20210617161348 do + let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } + let!(:project) { table(:projects).create!(namespace_id: namespace.id) } + let!(:environment) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) } + let(:background_migration_jobs) { table(:background_migration_jobs) } + + before do + create_deployment!(environment.id, project.id) + create_deployment!(environment.id, project.id) + create_deployment!(environment.id, project.id) + create_deployment!(non_existing_record_id, project.id) + create_deployment!(non_existing_record_id, project.id) + create_deployment!(non_existing_record_id, project.id) + create_deployment!(non_existing_record_id, project.id) + + stub_const("#{described_class}::BATCH_SIZE", 1) + end + + it 'schedules DeleteOrphanedDeployments background jobs' do + Sidekiq::Testing.fake! do + freeze_time do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq(7) + table(:deployments).find_each do |deployment| + expect(described_class::MIGRATION).to be_scheduled_migration(deployment.id, deployment.id) + end + end + end + end + + def create_deployment!(environment_id, project_id) + table(:deployments).create!( + environment_id: environment_id, + project_id: project_id, + ref: 'master', + tag: false, + sha: 'x', + status: 1, + iid: table(:deployments).count + 1) + end +end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 6290006b01b..1fbfae891af 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2172,15 +2172,15 @@ RSpec.describe Ci::Build do end it 'contains options' do - expect(build.options).to eq(options.stringify_keys) + expect(build.options).to eq(options.symbolize_keys) end - it 'allows to access with keys' do + it 'allows to access with symbolized keys' do expect(build.options[:image]).to eq('ruby:2.7') end - it 'allows to access with strings' do - expect(build.options['image']).to eq('ruby:2.7') + it 'rejects access with string keys' do + expect(build.options['image']).to be_nil end context 'when ci_build_metadata_config is set' do @@ -2189,7 +2189,7 @@ RSpec.describe Ci::Build do end it 'persist data in build metadata' do - expect(build.metadata.read_attribute(:config_options)).to eq(options.stringify_keys) + expect(build.metadata.read_attribute(:config_options)).to eq(options.symbolize_keys) end it 'does not persist data in build' do @@ -4715,9 +4715,9 @@ RSpec.describe Ci::Build do describe '#read_metadata_attribute' do let(:build) { create(:ci_build, :degenerated) } - let(:build_options) { { "key" => "build" } } - let(:metadata_options) { { "key" => "metadata" } } - let(:default_options) { { "key" => "default" } } + let(:build_options) { { key: "build" } } + let(:metadata_options) { { key: "metadata" } } + let(:default_options) { { key: "default" } } subject { build.send(:read_metadata_attribute, :options, :config_options, default_options) } @@ -4752,8 +4752,8 @@ RSpec.describe Ci::Build do describe '#write_metadata_attribute' do let(:build) { create(:ci_build, :degenerated) } - let(:options) { { "key" => "new options" } } - let(:existing_options) { { "key" => "existing options" } } + let(:options) { { key: "new options" } } + let(:existing_options) { { key: "existing options" } } subject { build.send(:write_metadata_attribute, :options, :config_options, options) } diff --git a/spec/models/ci/job_token/scope_spec.rb b/spec/models/ci/job_token/scope_spec.rb index c731a2634f5..37d3ec729f8 100644 --- a/spec/models/ci/job_token/scope_spec.rb +++ b/spec/models/ci/job_token/scope_spec.rb @@ -29,7 +29,7 @@ RSpec.describe Ci::JobToken::Scope do end end - describe 'includes?' do + describe '#includes?' do subject { scope.includes?(target_project) } context 'when param is the project defining the scope' do diff --git a/spec/requests/api/graphql/mutations/ci/job_token_scope/add_project_spec.rb b/spec/requests/api/graphql/mutations/ci/job_token_scope/add_project_spec.rb new file mode 100644 index 00000000000..926c310cad6 --- /dev/null +++ b/spec/requests/api/graphql/mutations/ci/job_token_scope/add_project_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'CiJobTokenScopeAddProject' do + include GraphqlHelpers + + let_it_be(:project) { create(:project) } + let_it_be(:target_project) { create(:project) } + + let(:variables) do + { + project_path: project.full_path, + target_project_path: target_project.full_path + } + end + + let(:mutation) do + graphql_mutation(:ci_job_token_scope_add_project, variables) do + <<~QL + errors + ciJobTokenScope { + projects { + nodes { + path + } + } + } + QL + end + end + + let(:mutation_response) { graphql_mutation_response(:ci_job_token_scope_add_project) } + + context 'when unauthorized' do + let(:current_user) { create(:user) } + + context 'when not a maintainer' do + before do + project.add_developer(current_user) + end + + it 'has graphql errors' do + post_graphql_mutation(mutation, current_user: current_user) + + expect(graphql_errors).not_to be_empty + end + end + end + + context 'when authorized' do + let_it_be(:current_user) { project.owner } + + before do + target_project.add_developer(current_user) + end + + it 'adds the target project to the job token scope' do + expect do + post_graphql_mutation(mutation, current_user: current_user) + expect(response).to have_gitlab_http_status(:success) + expect(mutation_response.dig('ciJobTokenScope', 'projects', 'nodes')).not_to be_empty + end.to change { Ci::JobToken::Scope.new(project).includes?(target_project) }.from(false).to(true) + end + + context 'when invalid target project is provided' do + before do + variables[:target_project_path] = 'unknown/project' + end + + it 'has mutation errors' do + post_graphql_mutation(mutation, current_user: current_user) + + expect(mutation_response['errors']).to contain_exactly(Ci::JobTokenScope::AddProjectService::TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND) + end + end + end +end diff --git a/spec/services/ci/archive_trace_service_spec.rb b/spec/services/ci/archive_trace_service_spec.rb index a4f498f17c3..3ec671d6add 100644 --- a/spec/services/ci/archive_trace_service_spec.rb +++ b/spec/services/ci/archive_trace_service_spec.rb @@ -30,43 +30,17 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do create(:ci_build_trace_chunk, build: job) end - context 'when the feature flag `erase_traces_from_already_archived_jobs_when_archiving_again` is enabled' do - before do - stub_feature_flags(erase_traces_from_already_archived_jobs_when_archiving_again: true) - end - - it 'removes the trace chunks' do - expect { subject }.to change { job.trace_chunks.count }.to(0) - end - - context 'when associated data does not exist' do - before do - job.job_artifacts_trace.file.remove! - end - - it 'removes the trace artifact' do - expect { subject }.to change { job.reload.job_artifacts_trace }.to(nil) - end - end + it 'removes the trace chunks' do + expect { subject }.to change { job.trace_chunks.count }.to(0) end - context 'when the feature flag `erase_traces_from_already_archived_jobs_when_archiving_again` is disabled' do + context 'when associated data does not exist' do before do - stub_feature_flags(erase_traces_from_already_archived_jobs_when_archiving_again: false) + job.job_artifacts_trace.file.remove! end - it 'does not remove the trace chunks' do - expect { subject }.not_to change { job.trace_chunks.count } - end - - context 'when associated data does not exist' do - before do - job.job_artifacts_trace.file.remove! - end - - it 'does not remove the trace artifact' do - expect { subject }.not_to change { job.reload.job_artifacts_trace } - end + it 'removes the trace artifact' do + expect { subject }.to change { job.reload.job_artifacts_trace }.to(nil) end end end diff --git a/spec/services/ci/create_pipeline_service/cache_spec.rb b/spec/services/ci/create_pipeline_service/cache_spec.rb index 5f74c2f1cef..f9767a794db 100644 --- a/spec/services/ci/create_pipeline_service/cache_spec.rb +++ b/spec/services/ci/create_pipeline_service/cache_spec.rb @@ -33,11 +33,11 @@ RSpec.describe Ci::CreatePipelineService do it 'uses the provided key' do expected = { - 'key' => 'a-key', - 'paths' => ['logs/', 'binaries/'], - 'policy' => 'pull-push', - 'untracked' => true, - 'when' => 'on_success' + key: 'a-key', + paths: ['logs/', 'binaries/'], + policy: 'pull-push', + untracked: true, + when: 'on_success' } expect(pipeline).to be_persisted @@ -66,10 +66,10 @@ RSpec.describe Ci::CreatePipelineService do it 'builds a cache key' do expected = { - 'key' => /[a-f0-9]{40}/, - 'paths' => ['logs/'], - 'policy' => 'pull-push', - 'when' => 'on_success' + key: /[a-f0-9]{40}/, + paths: ['logs/'], + policy: 'pull-push', + when: 'on_success' } expect(pipeline).to be_persisted @@ -82,10 +82,10 @@ RSpec.describe Ci::CreatePipelineService do it 'uses default cache key' do expected = { - 'key' => /default/, - 'paths' => ['logs/'], - 'policy' => 'pull-push', - 'when' => 'on_success' + key: /default/, + paths: ['logs/'], + policy: 'pull-push', + when: 'on_success' } expect(pipeline).to be_persisted @@ -115,10 +115,10 @@ RSpec.describe Ci::CreatePipelineService do it 'builds a cache key' do expected = { - 'key' => /\$ENV_VAR-[a-f0-9]{40}/, - 'paths' => ['logs/'], - 'policy' => 'pull-push', - 'when' => 'on_success' + key: /\$ENV_VAR-[a-f0-9]{40}/, + paths: ['logs/'], + policy: 'pull-push', + when: 'on_success' } expect(pipeline).to be_persisted @@ -131,10 +131,10 @@ RSpec.describe Ci::CreatePipelineService do it 'uses default cache key' do expected = { - 'key' => /\$ENV_VAR-default/, - 'paths' => ['logs/'], - 'policy' => 'pull-push', - 'when' => 'on_success' + key: /\$ENV_VAR-default/, + paths: ['logs/'], + policy: 'pull-push', + when: 'on_success' } expect(pipeline).to be_persisted diff --git a/spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb b/spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb index 4cf52223e38..5dceb9f57f0 100644 --- a/spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb +++ b/spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb @@ -39,8 +39,8 @@ RSpec.describe Ci::CreatePipelineService do it 'creates a pipeline' do expect(pipeline).to be_persisted expect(pipeline.builds.first.options).to match(a_hash_including({ - 'before_script' => ['ls'], - 'script' => [ + before_script: ['ls'], + script: [ 'echo doing my first step', 'echo doing step 1 of job 1', 'echo doing my last step' diff --git a/spec/services/ci/create_pipeline_service/needs_spec.rb b/spec/services/ci/create_pipeline_service/needs_spec.rb index 3b4a6178b8f..3246a39e88b 100644 --- a/spec/services/ci/create_pipeline_service/needs_spec.rb +++ b/spec/services/ci/create_pipeline_service/needs_spec.rb @@ -104,7 +104,7 @@ RSpec.describe Ci::CreatePipelineService do it 'saves dependencies' do expect(test_a_build.options) - .to match(a_hash_including('dependencies' => ['build_a'])) + .to match(a_hash_including(dependencies: ['build_a'])) end it 'artifacts default to true' do diff --git a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb index 512cf546e6a..1164d344a79 100644 --- a/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb +++ b/spec/services/ci/create_pipeline_service/parent_child_pipeline_spec.rb @@ -69,9 +69,9 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => [ - { 'local' => 'path/to/child.yml' } + trigger: { + include: [ + { local: 'path/to/child.yml' } ] } } @@ -149,9 +149,9 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => [ - { 'local' => 'path/to/child.yml' } + trigger: { + include: [ + { local: 'path/to/child.yml' } ] } } @@ -175,8 +175,8 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => 'path/to/child.yml' + trigger: { + include: 'path/to/child.yml' } } end @@ -202,8 +202,8 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => ['path/to/child.yml', 'path/to/child2.yml'] + trigger: { + include: ['path/to/child.yml', 'path/to/child2.yml'] } } end @@ -295,12 +295,12 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => [ + trigger: { + include: [ { - 'file' => 'path/to/child.yml', - 'project' => 'my-namespace/my-project', - 'ref' => 'master' + file: 'path/to/child.yml', + project: 'my-namespace/my-project', + ref: 'master' } ] } @@ -353,11 +353,11 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do it_behaves_like 'successful creation' do let(:expected_bridge_options) do { - 'trigger' => { - 'include' => [ + trigger: { + include: [ { - 'file' => ["path/to/child1.yml", "path/to/child2.yml"], - 'project' => 'my-namespace/my-project' + file: ["path/to/child1.yml", "path/to/child2.yml"], + project: 'my-namespace/my-project' } ] } diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 3316f8c3d9b..c27088f805f 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -1001,7 +1001,7 @@ RSpec.describe Ci::CreatePipelineService do expect(pipeline.yaml_errors).not_to be_present expect(pipeline).to be_persisted expect(build).to be_kind_of(Ci::Build) - expect(build.options).to eq(config[:release].except(:stage, :only).with_indifferent_access) + expect(build.options).to eq(config[:release].except(:stage, :only)) expect(build).to be_persisted end end diff --git a/spec/services/ci/job_token_scope/add_project_service_spec.rb b/spec/services/ci/job_token_scope/add_project_service_spec.rb new file mode 100644 index 00000000000..37134af7c2a --- /dev/null +++ b/spec/services/ci/job_token_scope/add_project_service_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Ci::JobTokenScope::AddProjectService do + let(:service) { described_class.new(project, current_user) } + + let_it_be(:project) { create(:project) } + let_it_be(:target_project) { create(:project) } + let_it_be(:current_user) { create(:user) } + + describe '#execute' do + subject(:result) { service.execute(target_project) } + + shared_examples 'returns error' do |error| + it 'returns an error response', :aggregate_failures do + expect(result).to be_error + expect(result.message).to eq(error) + end + end + + context 'when job token scope is disabled for the given project' do + before do + allow(project).to receive(:ci_job_token_scope_enabled?).and_return(false) + end + + it_behaves_like 'returns error', 'Job token scope is disabled for this project' + end + + context 'when user does not have permissions to edit the job token scope' do + it_behaves_like 'returns error', 'Insufficient permissions to modify the job token scope' + end + + context 'when user has permissions to edit the job token scope' do + before do + project.add_maintainer(current_user) + end + + context 'when target project is not provided' do + let(:target_project) { nil } + + it_behaves_like 'returns error', Ci::JobTokenScope::AddProjectService::TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND + end + + context 'when target project is provided' do + context 'when user does not have permissions to read the target project' do + it_behaves_like 'returns error', Ci::JobTokenScope::AddProjectService::TARGET_PROJECT_UNAUTHORIZED_OR_UNFOUND + end + + context 'when user has permissions to read the target project' do + before do + target_project.add_guest(current_user) + end + + it 'adds the project to the scope' do + expect do + expect(result).to be_success + end.to change { Ci::JobToken::ProjectScopeLink.count }.by(1) + end + + context 'when target project is already in scope' do + before do + create(:ci_job_token_project_scope_link, + source_project: project, + target_project: target_project) + end + + it_behaves_like 'returns error', "Target project is already in the job token scope" + end + end + + context 'when target project is same as the source project' do + let(:target_project) { project } + + it_behaves_like 'returns error', "Validation failed: Target project can't be the same as the source project" + end + end + end + end +end diff --git a/spec/services/ci/register_job_service_spec.rb b/spec/services/ci/register_job_service_spec.rb index 6186a017eb5..5b215db8570 100644 --- a/spec/services/ci/register_job_service_spec.rb +++ b/spec/services/ci/register_job_service_spec.rb @@ -145,7 +145,7 @@ module Ci context 'when using DEFCON mode that disables fair scheduling' do before do - stub_feature_flags(ci_queueing_disaster_recovery: true) + stub_feature_flags(ci_queueing_disaster_recovery_disable_fair_scheduling: true) end context 'when all builds are pending' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6d0969e8688..7383fc91bc1 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -260,8 +260,9 @@ RSpec.configure do |config| # tests, until we introduce it in user settings stub_feature_flags(forti_token_cloud: false) - # This feature flag is by default disabled and used in disaster recovery mode - stub_feature_flags(ci_queueing_disaster_recovery: false) + # These feature flag are by default disabled and used in disaster recovery mode + stub_feature_flags(ci_queueing_disaster_recovery_disable_fair_scheduling: false) + stub_feature_flags(ci_queueing_disaster_recovery_disable_quota: false) enable_rugged = example.metadata[:enable_rugged].present? |