diff options
39 files changed, 516 insertions, 171 deletions
diff --git a/.rubocop_todo/database/multiple_databases.yml b/.rubocop_todo/database/multiple_databases.yml index 40de71b1203..1c63431c4d7 100644 --- a/.rubocop_todo/database/multiple_databases.yml +++ b/.rubocop_todo/database/multiple_databases.yml @@ -16,7 +16,6 @@ Database/MultipleDatabases: - 'lib/backup/manager.rb' - lib/gitlab/background_migration/backfill_projects_with_coverage.rb - lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb - - lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb - 'lib/gitlab/database.rb' - 'lib/gitlab/database/load_balancing/load_balancer.rb' - 'lib/gitlab/database/migrations/observers/query_log.rb' diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index c1b865ae578..5409bdf5af4 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -2,6 +2,8 @@ module ContainerRegistry class Event + include Gitlab::Utils::StrongMemoize + ALLOWED_ACTIONS = %w(push delete).freeze PUSH_ACTION = 'push' EVENT_TRACKING_CATEGORY = 'container_registry:notification' @@ -17,7 +19,7 @@ module ContainerRegistry end def handle! - # no op + update_project_statistics end def track! @@ -58,10 +60,25 @@ module ContainerRegistry end def container_registry_path - path = event.dig('target', 'repository') - return unless path + strong_memoize(:container_registry_path) do + path = event.dig('target', 'repository') + next unless path + + ContainerRegistry::Path.new(path) + end + end + + def project + container_registry_path&.repository_project + end + + def update_project_statistics + return unless supported? + return unless target_tag? + return unless project + return unless Feature.enabled?(:container_registry_project_statistics, project) - ContainerRegistry::Path.new(path) + ProjectCacheWorker.perform_async(project.id, [], [:container_registry_size]) end end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index e8b6cf00d17..de196964d31 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -44,6 +44,7 @@ class Namespace < ApplicationRecord has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true + has_one :ci_cd_settings, inverse_of: :namespace, class_name: 'NamespaceCiCdSetting', autosave: true has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' diff --git a/app/models/namespace_ci_cd_setting.rb b/app/models/namespace_ci_cd_setting.rb new file mode 100644 index 00000000000..0e044f4fb10 --- /dev/null +++ b/app/models/namespace_ci_cd_setting.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class NamespaceCiCdSetting < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass + belongs_to :namespace, inverse_of: :ci_cd_settings + + self.primary_key = :namespace_id +end diff --git a/config/application.rb b/config/application.rb index 164019f9f3b..3670d787fbd 100644 --- a/config/application.rb +++ b/config/application.rb @@ -72,7 +72,7 @@ module Gitlab require_dependency Rails.root.join('lib/gitlab/patch/database_config') require_dependency Rails.root.join('lib/gitlab/exceptions_app') - config.exceptions_app = Gitlab::ExceptionsApp.new(Rails.public_path) + config.exceptions_app = Gitlab::ExceptionsApp.new(Gitlab.jh? ? Rails.root.join('jh/public') : Rails.public_path) # This preload is required to: # diff --git a/config/feature_flags/development/ci_variables_builder_config_variables.yml b/config/feature_flags/development/ci_variables_builder_config_variables.yml deleted file mode 100644 index eb74993f90f..00000000000 --- a/config/feature_flags/development/ci_variables_builder_config_variables.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_variables_builder_config_variables -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79935 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/358791 -milestone: '14.10' -type: development -group: group::pipeline execution -default_enabled: false diff --git a/db/docs/namespace_ci_cd_settings.yml b/db/docs/namespace_ci_cd_settings.yml new file mode 100644 index 00000000000..8159f721c98 --- /dev/null +++ b/db/docs/namespace_ci_cd_settings.yml @@ -0,0 +1,10 @@ +--- +table_name: namespace_ci_cd_settings +classes: +- NamespaceCiCdSetting +feature_categories: +- continuous_integration +- runner_fleet +description: Namespace-scoped settings related to the CI/CD domain +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86477 +milestone: '15.0' diff --git a/db/migrate/20220503102855_add_namespace_ci_cd_settings_table.rb b/db/migrate/20220503102855_add_namespace_ci_cd_settings_table.rb new file mode 100644 index 00000000000..532e80d1567 --- /dev/null +++ b/db/migrate/20220503102855_add_namespace_ci_cd_settings_table.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddNamespaceCiCdSettingsTable < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def up + create_table :namespace_ci_cd_settings, id: false do |t| + t.references :namespace, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade } + t.boolean :allow_stale_runner_pruning, null: false, default: false + end + end + + def down + drop_table :namespace_ci_cd_settings, if_exists: true + end +end diff --git a/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb b/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb index a3e59b38975..9ce973183eb 100644 --- a/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb +++ b/db/post_migrate/20220502015011_clean_up_fix_merge_request_diff_commit_users.rb @@ -3,6 +3,8 @@ class CleanUpFixMergeRequestDiffCommitUsers < Gitlab::Database::Migration[2.0] disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + MIGRATION_CLASS = 'FixMergeRequestDiffCommitUsers' def up diff --git a/db/post_migrate/20220504083348_add_indexes_for_primary_email_post_regression_cleanup_migration.rb b/db/post_migrate/20220504083348_add_indexes_for_primary_email_post_regression_cleanup_migration.rb new file mode 100644 index 00000000000..ba0457fa36e --- /dev/null +++ b/db/post_migrate/20220504083348_add_indexes_for_primary_email_post_regression_cleanup_migration.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class AddIndexesForPrimaryEmailPostRegressionCleanupMigration < Gitlab::Database::Migration[2.0] + USERS_INDEX = :index_users_on_id_for_primary_email_migration + EMAIL_INDEX = :index_emails_on_email_user_id + + disable_ddl_transaction! + + def up + unless index_exists_by_name?(:users, USERS_INDEX) + + disable_statement_timeout do + execute <<~SQL + CREATE INDEX CONCURRENTLY #{USERS_INDEX} + ON users (id) INCLUDE (email, confirmed_at) + WHERE confirmed_at IS NOT NULL + SQL + end + end + + add_concurrent_index :emails, [:email, :user_id], name: EMAIL_INDEX + end + + def down + remove_concurrent_index_by_name :users, USERS_INDEX + remove_concurrent_index_by_name :emails, EMAIL_INDEX + end +end diff --git a/db/post_migrate/20220504083836_cleanup_after_fixing_regression_with_new_users_emails.rb b/db/post_migrate/20220504083836_cleanup_after_fixing_regression_with_new_users_emails.rb new file mode 100644 index 00000000000..110e2578cd5 --- /dev/null +++ b/db/post_migrate/20220504083836_cleanup_after_fixing_regression_with_new_users_emails.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +class CleanupAfterFixingRegressionWithNewUsersEmails < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + BATCH_SIZE = 10_000 + + # Stubbed class to access the User table + class User < MigrationRecord + include ::EachBatch + + self.table_name = 'users' + self.inheritance_column = :_type_disabled + + scope :confirmed, -> { where.not(confirmed_at: nil) } + + has_many :emails + end + + # Stubbed class to access the Emails table + class Email < MigrationRecord + self.table_name = 'emails' + self.inheritance_column = :_type_disabled + + belongs_to :user + end + + # rubocop: disable Layout/LineLength + def up + # Select confirmed users that do not have their primary email in the emails table, + # and create the email record. + not_exists_condition = 'NOT EXISTS (SELECT 1 FROM emails WHERE emails.email = users.email AND emails.user_id = users.id)' + + User.confirmed.each_batch(of: BATCH_SIZE) do |user_batch| + user_batch.select(:id, :email, :confirmed_at).where(not_exists_condition).each do |user| + current_time = Time.now.utc + + begin + Email.create( + user_id: user.id, + email: user.email, + confirmed_at: user.confirmed_at, + created_at: current_time, + updated_at: current_time + ) + rescue StandardError => error + Gitlab::AppLogger.error("Could not add primary email #{user.email} to emails for user with ID #{user.id} due to #{error}") + end + end + end + end + # rubocop: enable Layout/LineLength + + def down + # Intentionally left blank + end +end diff --git a/db/post_migrate/20220504084136_drop_temporary_indexes_for_primary_email_post_regression_cleanup.rb b/db/post_migrate/20220504084136_drop_temporary_indexes_for_primary_email_post_regression_cleanup.rb new file mode 100644 index 00000000000..60c69426a8a --- /dev/null +++ b/db/post_migrate/20220504084136_drop_temporary_indexes_for_primary_email_post_regression_cleanup.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class DropTemporaryIndexesForPrimaryEmailPostRegressionCleanup < Gitlab::Database::Migration[2.0] + USERS_INDEX = :index_users_on_id_for_primary_email_migration + EMAIL_INDEX = :index_emails_on_email_user_id + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name :users, USERS_INDEX + remove_concurrent_index_by_name :emails, EMAIL_INDEX + end + + def down + unless index_exists_by_name?(:users, USERS_INDEX) + + disable_statement_timeout do + execute <<~SQL + CREATE INDEX CONCURRENTLY #{USERS_INDEX} + ON users (id) INCLUDE (email, confirmed_at) + WHERE confirmed_at IS NOT NULL + SQL + end + end + + add_concurrent_index :emails, [:email, :user_id], name: EMAIL_INDEX + end +end diff --git a/db/schema_migrations/20220503102855 b/db/schema_migrations/20220503102855 new file mode 100644 index 00000000000..e63b6ebe429 --- /dev/null +++ b/db/schema_migrations/20220503102855 @@ -0,0 +1 @@ +e573f8f4ebca77bae3f3ecc68d4e5a9fe015f4a1040e72615b74cc158a7f7e5f
\ No newline at end of file diff --git a/db/schema_migrations/20220504083348 b/db/schema_migrations/20220504083348 new file mode 100644 index 00000000000..ef9c042e036 --- /dev/null +++ b/db/schema_migrations/20220504083348 @@ -0,0 +1 @@ +bb01480d2108f02bb6adcc671afc40de88655644e78397f1395fbb5af32cd24a
\ No newline at end of file diff --git a/db/schema_migrations/20220504083836 b/db/schema_migrations/20220504083836 new file mode 100644 index 00000000000..cfde07ab90d --- /dev/null +++ b/db/schema_migrations/20220504083836 @@ -0,0 +1 @@ +686d7d3266e7df938827f34a65fe73c0b92d0a76df34789751e609627a0768f6
\ No newline at end of file diff --git a/db/schema_migrations/20220504084136 b/db/schema_migrations/20220504084136 new file mode 100644 index 00000000000..49ed93714c2 --- /dev/null +++ b/db/schema_migrations/20220504084136 @@ -0,0 +1 @@ +6d4b1755e342c6608b95d7abca028353d2f0e84efb63e06b9cc230c23f737565
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index ebbee9ec465..5fea5ed9391 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -17359,6 +17359,11 @@ CREATE TABLE namespace_aggregation_schedules ( namespace_id integer NOT NULL ); +CREATE TABLE namespace_ci_cd_settings ( + namespace_id bigint NOT NULL, + allow_stale_runner_pruning boolean DEFAULT false NOT NULL +); + CREATE TABLE namespace_limits ( additional_purchased_storage_size bigint DEFAULT 0 NOT NULL, additional_purchased_storage_ends_on date, @@ -24889,6 +24894,9 @@ ALTER TABLE ONLY namespace_admin_notes ALTER TABLE ONLY namespace_aggregation_schedules ADD CONSTRAINT namespace_aggregation_schedules_pkey PRIMARY KEY (namespace_id); +ALTER TABLE ONLY namespace_ci_cd_settings + ADD CONSTRAINT namespace_ci_cd_settings_pkey PRIMARY KEY (namespace_id); + ALTER TABLE ONLY namespace_limits ADD CONSTRAINT namespace_limits_pkey PRIMARY KEY (namespace_id); @@ -33149,6 +33157,9 @@ ALTER TABLE ONLY vulnerability_occurrence_identifiers ALTER TABLE ONLY alert_management_http_integrations ADD CONSTRAINT fk_rails_bec49f52cc FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY namespace_ci_cd_settings + ADD CONSTRAINT fk_rails_bf04185d54 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_occurrences ADD CONSTRAINT fk_rails_bf5b788ca7 FOREIGN KEY (scanner_id) REFERENCES vulnerability_scanners(id) ON DELETE CASCADE; diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index caeeb12fd35..55f8321b559 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -66,7 +66,7 @@ configure the infrastructure for your application. | Topic | Description | Good for beginners | |-------|-------------|--------------------| -| [Connect with a Kubernetes cluster](https://about.gitlab.com/blog/2021/11/18/gitops-with-gitlab-connecting-the-cluster/) | Connect a Kubernetes cluster with GitLab for pull and push based deployments and security integrations. | | +| [Use GitOps with GitLab](https://about.gitlab.com/blog/2022/04/07/the-ultimate-guide-to-gitops-with-gitlab/) | Learn how to provision and manage a base infrastructure, connect to a Kubernetes cluster, deploy third-party applications, and carry out other infrastructure automation tasks. | | | [Use Auto DevOps to deploy an application](../topics/autodevops/quick_start_guide.md) | Deploy an application to Google Kubernetes Engine (GKE). | | ## Publish a static website diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 7cbb0c08720..0442d7d13f0 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -53,7 +53,7 @@ For example, consider a user named Alex: | GitLab URL | Namespace | | ---------- | --------- | | Alex creates an account with the username `alex`: `https://gitlab.example.com/alex`. | The namespace in this case is `alex`. | -| Alex creates a group for their team with the group name `alex-team`. The group and its projects are available at: `https://gitlab.example.com/alex-team`. | The namespace in this cases is `alex-team`. | +| Alex creates a group for their team with the group name `alex-team`. The group and its projects are available at: `https://gitlab.example.com/alex-team`. | The namespace in this case is `alex-team`. | | Alex creates a subgroup of `alex-team` with the subgroup name `marketing`. The subgroup and its projects are available at: `https://gitlab.example.com/alex-team/marketing`. | The namespace in this case is `alex-team/marketing`. | ## Create a group diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb index 78e897d9ae1..36d4e649271 100644 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -26,7 +26,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ::Ci::ApplicationRecord.connection end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 783a7fe28db..15a4ff91c1b 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -26,11 +26,8 @@ module Gitlab @source_ref_path = pipeline&.source_ref_path @project = project - if use_config_variables? - pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - end - @context = self.logger.instrument(:config_build_context) do + pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) end @@ -149,46 +146,15 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline), + variables: build_variables(pipeline: pipeline), logger: logger) end - def build_variables(project:, pipeline:) + def build_variables(pipeline:) logger.instrument(:config_build_variables) do - build_variables_without_instrumentation( - project: project, - pipeline: pipeline - ) - end - end - - def build_variables_without_instrumentation(project:, pipeline:) - if use_config_variables? - return pipeline.variables_builder.config_variables - end - - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless project - - # The order of the following lines is important as priority of CI variables is - # defined globally within GitLab. - # - # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence - variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) if pipeline - variables.concat(secret_variables(project: project, pipeline: pipeline)) - variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group - variables.concat(project.ci_variables_for(ref: source_ref_path)) - variables.concat(pipeline.variables) if pipeline - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule - end - end - - def secret_variables(project:, pipeline:) - if pipeline - pipeline.variables_builder.secret_instance_variables - else - Gitlab::Ci::Variables::Builder::Instance.new.secret_variables + pipeline + .variables_builder + .config_variables end end @@ -196,12 +162,6 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end - def use_config_variables? - strong_memoize(:use_config_variables) do - ::Feature.enabled?(:ci_variables_builder_config_variables, @project) - end - end - # Overridden in EE def rescue_errors RESCUE_ERRORS diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index f769201464f..900ba1e5b36 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -327,6 +327,7 @@ namespace_aggregation_schedules: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main +namespace_ci_cd_settings: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main namespaces_sync_events: :gitlab_main diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 2321d37aa26..95d4a7c67f7 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -325,11 +325,6 @@ msgid_plural "%d more comments" msgstr[0] "" msgstr[1] "" -msgid "%d new license" -msgid_plural "%d new licenses" -msgstr[0] "" -msgstr[1] "" - msgid "%d open issue" msgid_plural "%d open issues" msgstr[0] "" @@ -365,11 +360,6 @@ msgid_plural "%d projects selected" msgstr[0] "" msgstr[1] "" -msgid "%d removed license" -msgid_plural "%d removed licenses" -msgstr[0] "" -msgstr[1] "" - msgid "%d second" msgid_plural "%d seconds" msgstr[0] "" @@ -4601,33 +4591,12 @@ msgid_plural "ApprovalRuleSummary|%{count} approvals required from %{membersCoun msgstr[0] "" msgstr[1] "" -msgid "ApprovalRule|%{firstLabel} +%{numberOfAdditionalLabels} more" -msgstr "" - msgid "ApprovalRule|A merge request author collaborating with a merge request approver" msgstr "" msgid "ApprovalRule|Add approvers" msgstr "" -msgid "ApprovalRule|All scanners" -msgstr "" - -msgid "ApprovalRule|All severity levels" -msgstr "" - -msgid "ApprovalRule|All vulnerability states" -msgstr "" - -msgid "ApprovalRule|Apply this approval rule to consider only the selected security scanners." -msgstr "" - -msgid "ApprovalRule|Apply this approval rule to consider only the selected severity levels." -msgstr "" - -msgid "ApprovalRule|Apply this approval rule to consider only the selected vulnerability states." -msgstr "" - msgid "ApprovalRule|Approval rules" msgstr "" @@ -4664,21 +4633,6 @@ msgstr "" msgid "ApprovalRule|Newly detected" msgstr "" -msgid "ApprovalRule|Number of vulnerabilities allowed before approval rule is triggered." -msgstr "" - -msgid "ApprovalRule|Please enter a number equal or greater than zero" -msgstr "" - -msgid "ApprovalRule|Please select at least one security scanner" -msgstr "" - -msgid "ApprovalRule|Please select at least one severity level" -msgstr "" - -msgid "ApprovalRule|Please select at least one vulnerability state" -msgstr "" - msgid "ApprovalRule|Previously detected" msgstr "" @@ -4691,39 +4645,15 @@ msgstr "" msgid "ApprovalRule|Rule name" msgstr "" -msgid "ApprovalRule|Security scanners" -msgstr "" - -msgid "ApprovalRule|Select All" -msgstr "" - msgid "ApprovalRule|Select eligible approvers by expertise or files changed." msgstr "" -msgid "ApprovalRule|Select scanners" -msgstr "" - -msgid "ApprovalRule|Select severity levels" -msgstr "" - -msgid "ApprovalRule|Select vulnerability states" -msgstr "" - -msgid "ApprovalRule|Severity levels" -msgstr "" - msgid "ApprovalRule|Target branch" msgstr "" msgid "ApprovalRule|Try for free" msgstr "" -msgid "ApprovalRule|Vulnerabilities allowed" -msgstr "" - -msgid "ApprovalRule|Vulnerability states" -msgstr "" - msgid "ApprovalSettings|Merge request approval settings have been updated." msgstr "" @@ -22611,11 +22541,6 @@ msgid_plural "LicenseCompliance|License Compliance detected %d new licenses and msgstr[0] "" msgstr[1] "" -msgid "LicenseCompliance|License Compliance detected %d removed license" -msgid_plural "LicenseCompliance|License Compliance detected %d removed licenses" -msgstr[0] "" -msgstr[1] "" - msgid "LicenseCompliance|License Compliance detected no licenses for the source branch only" msgstr "" @@ -33317,9 +33242,6 @@ msgstr "" msgid "Security report is out of date. Run %{newPipelineLinkStart}a new pipeline%{newPipelineLinkEnd} for the target branch (%{targetBranchName})" msgstr "" -msgid "SecurityApprovals|A merge request approval is required when a security report contains a new vulnerability." -msgstr "" - msgid "SecurityApprovals|A merge request approval is required when test coverage declines." msgstr "" @@ -33335,9 +33257,6 @@ msgstr "" msgid "SecurityApprovals|Learn more about License-Check" msgstr "" -msgid "SecurityApprovals|Learn more about Vulnerability-Check" -msgstr "" - msgid "SecurityApprovals|License-Check" msgstr "" @@ -33347,12 +33266,6 @@ msgstr "" msgid "SecurityApprovals|Requires approval for decreases in test coverage. %{linkStart}Learn more.%{linkEnd}" msgstr "" -msgid "SecurityApprovals|Requires approval for vulnerabilities. %{linkStart}Learn more.%{linkEnd}" -msgstr "" - -msgid "SecurityApprovals|Vulnerability-Check" -msgstr "" - msgid "SecurityConfiguration|%{featureName} merge request creation mutation failed" msgstr "" diff --git a/package.json b/package.json index 8908958f32e..fc6b3c8a4ff 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@gitlab/at.js": "1.5.7", "@gitlab/favicon-overlay": "2.0.0", "@gitlab/svgs": "2.11.0", - "@gitlab/ui": "39.6.0", + "@gitlab/ui": "40.0.0", "@gitlab/visual-review-tools": "1.7.1", "@rails/actioncable": "6.1.4-7", "@rails/ujs": "6.1.4-7", diff --git a/spec/controllers/projects/environments_controller_spec.rb b/spec/controllers/projects/environments_controller_spec.rb index f4cad5790a3..f63e0cea04c 100644 --- a/spec/controllers/projects/environments_controller_spec.rb +++ b/spec/controllers/projects/environments_controller_spec.rb @@ -208,6 +208,17 @@ RSpec.describe Projects::EnvironmentsController do expect(response).to have_gitlab_http_status(:not_found) end end + + it_behaves_like 'avoids N+1 queries on environment detail page' + + def create_deployment_with_associations(sequence:) + commit = project.commit("HEAD~#{sequence}") + create(:user, email: commit.author_email) + + deployer = create(:user) + build = create(:ci_build, environment: environment.name, pipeline: create(:ci_pipeline, project: environment.project), user: deployer) + create(:deployment, :success, environment: environment, deployable: build, user: deployer, project: project, sha: commit.sha) + end end describe 'GET edit' do diff --git a/spec/graphql/resolvers/issues_resolver_spec.rb b/spec/graphql/resolvers/issues_resolver_spec.rb index 3569244e514..e6ec9d8c895 100644 --- a/spec/graphql/resolvers/issues_resolver_spec.rb +++ b/spec/graphql/resolvers/issues_resolver_spec.rb @@ -631,13 +631,13 @@ RSpec.describe Resolvers::IssuesResolver do end it 'finds a specific issue with iid', :request_store do - result = batch_sync(max_queries: 6) { resolve_issues(iid: issue1.iid).to_a } + result = batch_sync(max_queries: 7) { resolve_issues(iid: issue1.iid).to_a } expect(result).to contain_exactly(issue1) end it 'batches queries that only include IIDs', :request_store do - result = batch_sync(max_queries: 6) do + result = batch_sync(max_queries: 7) do [issue1, issue2] .map { |issue| resolve_issues(iid: issue.iid.to_s) } .flat_map(&:to_a) @@ -647,7 +647,7 @@ RSpec.describe Resolvers::IssuesResolver do end it 'finds a specific issue with iids', :request_store do - result = batch_sync(max_queries: 6) do + result = batch_sync(max_queries: 7) do resolve_issues(iids: [issue1.iid]).to_a end diff --git a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb index 90dd3e14606..e38edfc3643 100644 --- a/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds_spec.rb @@ -5,9 +5,9 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, :migration, schema: 20220223112304 do let(:namespaces) { table(:namespaces) } let(:projects) { table(:projects) } - let(:ci_runners) { table(:ci_runners) } - let(:ci_pipelines) { table(:ci_pipelines) } - let(:ci_builds) { table(:ci_builds) } + let(:ci_runners) { table(:ci_runners, database: :ci) } + let(:ci_pipelines) { table(:ci_pipelines, database: :ci) } + let(:ci_builds) { table(:ci_builds, database: :ci) } subject { described_class.new } @@ -26,9 +26,9 @@ RSpec.describe Gitlab::BackgroundMigration::NullifyOrphanRunnerIdOnCiBuilds, :mi describe '#perform' do let(:namespace) { namespaces.create!(name: 'test', path: 'test', type: 'Group') } let(:project) { projects.create!(namespace_id: namespace.id, name: 'test') } - let(:pipeline) { ci_pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') } it 'nullifies runner_id for orphan ci_builds in range' do + pipeline = ci_pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') ci_runners.create!(id: 2, runner_type: 'project_type') ci_builds.create!(id: 5, type: 'Ci::Build', commit_id: pipeline.id, runner_id: 2) diff --git a/spec/migrations/20220502015011_clean_up_fix_merge_request_diff_commit_users_spec.rb b/spec/migrations/20220502015011_clean_up_fix_merge_request_diff_commit_users_spec.rb new file mode 100644 index 00000000000..769c0993b67 --- /dev/null +++ b/spec/migrations/20220502015011_clean_up_fix_merge_request_diff_commit_users_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! 'clean_up_fix_merge_request_diff_commit_users' + +RSpec.describe CleanUpFixMergeRequestDiffCommitUsers, :migration do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:project_namespace) { namespaces.create!(name: 'project2', path: 'project2', type: 'Project') } + let(:namespace) { namespaces.create!(name: 'foo', path: 'foo') } + + describe '#up' do + it 'finalizes the background migration' do + expect(described_class).to be_finalize_background_migration_of('FixMergeRequestDiffCommitUsers') + + migrate! + end + + it 'processes pending background jobs' do + project = projects.create!(name: 'p1', namespace_id: namespace.id, project_namespace_id: project_namespace.id) + + Gitlab::Database::BackgroundMigrationJob.create!( + class_name: 'FixMergeRequestDiffCommitUsers', + arguments: [project.id] + ) + + migrate! + + background_migrations = Gitlab::Database::BackgroundMigrationJob + .where(class_name: 'FixMergeRequestDiffCommitUsers') + + expect(background_migrations.count).to eq(0) + end + end +end diff --git a/spec/migrations/cleanup_after_fixing_regression_with_new_users_emails_spec.rb b/spec/migrations/cleanup_after_fixing_regression_with_new_users_emails_spec.rb new file mode 100644 index 00000000000..043bb091df3 --- /dev/null +++ b/spec/migrations/cleanup_after_fixing_regression_with_new_users_emails_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe CleanupAfterFixingRegressionWithNewUsersEmails, :sidekiq do + let(:migration) { described_class.new } + let(:users) { table(:users) } + let(:emails) { table(:emails) } + + # rubocop: disable Layout/LineLength + let!(:user_1) { users.create!(name: 'confirmed-user-1', email: 'confirmed-1@example.com', confirmed_at: 3.days.ago, projects_limit: 100) } + let!(:user_2) { users.create!(name: 'confirmed-user-2', email: 'confirmed-2@example.com', confirmed_at: 1.day.ago, projects_limit: 100) } + let!(:user_3) { users.create!(name: 'confirmed-user-3', email: 'confirmed-3@example.com', confirmed_at: 1.day.ago, projects_limit: 100) } + let!(:user_4) { users.create!(name: 'unconfirmed-user', email: 'unconfirmed@example.com', confirmed_at: nil, projects_limit: 100) } + + let!(:email_1) { emails.create!(email: 'confirmed-1@example.com', user_id: user_1.id, confirmed_at: 1.day.ago) } + let!(:email_2) { emails.create!(email: 'other_2@example.com', user_id: user_2.id, confirmed_at: 1.day.ago) } + + before do + stub_const("#{described_class.name}::BATCH_SIZE", 2) + end + + it 'adds primary email to emails for confirmed users that do not have their primary email in emails table', :aggregate_failures do + original_email_1_confirmed_at = email_1.reload.confirmed_at + + expect { migration.up }.to change { emails.count }.by(2) + + expect(emails.find_by(user_id: user_2.id, email: 'confirmed-2@example.com').confirmed_at).to eq(user_2.reload.confirmed_at) + expect(emails.find_by(user_id: user_3.id, email: 'confirmed-3@example.com').confirmed_at).to eq(user_3.reload.confirmed_at) + expect(email_1.reload.confirmed_at).to eq(original_email_1_confirmed_at) + + expect(emails.exists?(user_id: user_4.id)).to be(false) + end + # rubocop: enable Layout/LineLength + + it 'continues in case of errors with one email' do + allow(Email).to receive(:create) { raise 'boom!' } + + expect { migration.up }.not_to raise_error + end +end diff --git a/spec/models/container_registry/event_spec.rb b/spec/models/container_registry/event_spec.rb index 21a3ab5363a..6b544c95cc8 100644 --- a/spec/models/container_registry/event_spec.rb +++ b/spec/models/container_registry/event_spec.rb @@ -26,11 +26,65 @@ RSpec.describe ContainerRegistry::Event do end describe '#handle!' do - let(:raw_event) { { 'action' => 'push', 'target' => { 'mediaType' => ContainerRegistry::Client::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE } } } + let(:action) { 'push' } + let(:repository) { project.full_path } + let(:target) do + { + 'mediaType' => ContainerRegistry::Client::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, + 'tag' => 'latest', + 'repository' => repository + } + end + + let(:raw_event) { { 'action' => action, 'target' => target } } + + subject(:handle!) { described_class.new(raw_event).handle! } + + it 'enqueues a project statistics update' do + expect(ProjectCacheWorker).to receive(:perform_async).with(project.id, [], [:container_registry_size]) + + handle! + end - subject { described_class.new(raw_event).handle! } + shared_examples 'event without project statistics update' do + it 'does not queue a project statistics update' do + expect(ProjectCacheWorker).not_to receive(:perform_async) - it { is_expected.to eq nil } + handle! + end + end + + context 'with :container_registry_project_statistics feature flag disabled' do + before do + stub_feature_flags(container_registry_project_statistics: false) + end + + it_behaves_like 'event without project statistics update' + end + + context 'with no target tag' do + let(:target) { super().without('tag') } + + it_behaves_like 'event without project statistics update' + end + + context 'with an unsupported action' do + let(:action) { 'pull' } + + it_behaves_like 'event without project statistics update' + end + + context 'with an invalid project repository path' do + let(:repository) { 'does/not/exist' } + + it_behaves_like 'event without project statistics update' + end + + context 'with no project repository path' do + let(:repository) { nil } + + it_behaves_like 'event without project statistics update' + end end describe '#track!' do diff --git a/spec/models/namespace_ci_cd_setting_spec.rb b/spec/models/namespace_ci_cd_setting_spec.rb new file mode 100644 index 00000000000..9031d45221a --- /dev/null +++ b/spec/models/namespace_ci_cd_setting_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe NamespaceCiCdSetting do + describe "associations" do + it { is_expected.to belong_to(:namespace).inverse_of(:ci_cd_settings) } + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index b70d90d861b..abfde1291cf 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -32,6 +32,10 @@ RSpec.describe Namespace do it { is_expected.to have_one :namespace_route } it { is_expected.to have_many :namespace_members } + it do + is_expected.to have_one(:ci_cd_settings).class_name('NamespaceCiCdSetting').inverse_of(:namespace).autosave(true) + end + describe '#children' do let_it_be(:group) { create(:group) } let_it_be(:subgroup) { create(:group, parent: group) } diff --git a/spec/requests/api/container_registry_event_spec.rb b/spec/requests/api/container_registry_event_spec.rb index 4d38ddddffd..767e6e0b2ff 100644 --- a/spec/requests/api/container_registry_event_spec.rb +++ b/spec/requests/api/container_registry_event_spec.rb @@ -12,7 +12,7 @@ RSpec.describe API::ContainerRegistryEvent do allow(Gitlab.config.registry).to receive(:notification_secret) { secret_token } end - subject do + subject(:post_events) do post api('/container_registry_event/events'), params: { events: events }.to_json, headers: registry_headers.merge('Authorization' => secret_token) @@ -23,7 +23,7 @@ RSpec.describe API::ContainerRegistryEvent do allow(::ContainerRegistry::Event).to receive(:new).and_return(event) expect(event).to receive(:supported?).and_return(true) - subject + post_events expect(event).to have_received(:handle!).once expect(event).to have_received(:track!).once @@ -37,5 +37,37 @@ RSpec.describe API::ContainerRegistryEvent do expect(response).to have_gitlab_http_status(:unauthorized) end + + context 'when the event should update project statistics' do + let_it_be(:project) { create(:project) } + + let(:events) do + [ + { + action: 'push', + target: { + tag: 'latest', + repository: project.full_path + } + }, + { + action: 'delete', + target: { + tag: 'latest', + repository: project.full_path + } + } + ] + end + + it 'enqueues a project statistics update twice' do + expect(ProjectCacheWorker) + .to receive(:perform_async) + .with(project.id, [], [:container_registry_size]) + .twice.and_call_original + + expect { post_events }.to change { ProjectCacheWorker.jobs.size }.from(0).to(1) + end + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index cf2c0780298..e49e82f6ab6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -199,6 +199,7 @@ RSpec.configure do |config| config.include SidekiqMiddleware config.include StubActionCableConnection, type: :channel config.include StubSpamServices + config.include RenderedHelpers config.include RSpec::Benchmark::Matchers, type: :benchmark include StubFeatureFlags diff --git a/spec/support/helpers/migrations_helpers.rb b/spec/support/helpers/migrations_helpers.rb index afa7ee84bda..0c60871d2e7 100644 --- a/spec/support/helpers/migrations_helpers.rb +++ b/spec/support/helpers/migrations_helpers.rb @@ -1,12 +1,18 @@ # frozen_string_literal: true module MigrationsHelpers - def active_record_base - Gitlab::Database.database_base_models.fetch(self.class.metadata[:database] || :main) + def active_record_base(database: nil) + database_name = database || self.class.metadata[:database] || :main + + unless Gitlab::Database::DATABASE_NAMES.include?(database_name.to_s) + raise ArgumentError, "#{database_name} is not a valid argument" + end + + Gitlab::Database.database_base_models[database_name] || Gitlab::Database.database_base_models[:main] end - def table(name) - Class.new(active_record_base) do + def table(name, database: nil) + Class.new(active_record_base(database: database)) do self.table_name = name self.inheritance_column = :_type_disabled diff --git a/spec/support/helpers/rendered_helpers.rb b/spec/support/helpers/rendered_helpers.rb new file mode 100644 index 00000000000..137b7d5f708 --- /dev/null +++ b/spec/support/helpers/rendered_helpers.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module RenderedHelpers + # Wraps the `rendered` in `expect` to make it the target of an expectation. + # Designed to read nicely for one-liners. + # rubocop:disable RSpec/VoidExpect + def expect_rendered + render + expect(rendered) + end + # rubocop:enable RSpec/VoidExpect +end diff --git a/spec/support/shared_examples/controllers/environments_controller_shared_examples.rb b/spec/support/shared_examples/controllers/environments_controller_shared_examples.rb index c6e880635aa..a79b94209f3 100644 --- a/spec/support/shared_examples/controllers/environments_controller_shared_examples.rb +++ b/spec/support/shared_examples/controllers/environments_controller_shared_examples.rb @@ -65,3 +65,20 @@ RSpec.shared_examples 'failed response for #cancel_auto_stop' do end end end + +RSpec.shared_examples 'avoids N+1 queries on environment detail page' do + render_views + + before do + create_deployment_with_associations(sequence: 0) + end + + it 'avoids N+1 queries' do + control = ActiveRecord::QueryRecorder.new { get :show, params: environment_params } + + create_deployment_with_associations(sequence: 1) + create_deployment_with_associations(sequence: 2) + + expect { get :show, params: environment_params }.not_to exceed_query_limit(control.count).with_threshold(34) + end +end diff --git a/spec/support_specs/helpers/migrations_helpers_spec.rb b/spec/support_specs/helpers/migrations_helpers_spec.rb new file mode 100644 index 00000000000..b82eddad9bc --- /dev/null +++ b/spec/support_specs/helpers/migrations_helpers_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe MigrationsHelpers do + let(:helper_class) do + Class.new.tap do |klass| + klass.include described_class + allow(klass).to receive(:metadata).and_return(metadata) + end + end + + let(:metadata) { {} } + let(:helper) { helper_class.new } + + describe '#active_record_base' do + it 'returns the main base model' do + expect(helper.active_record_base).to eq(ActiveRecord::Base) + end + + context 'ci database configured' do + before do + skip_if_multiple_databases_not_setup + end + + it 'returns the CI base model' do + expect(helper.active_record_base(database: :ci)).to eq(Ci::ApplicationRecord) + end + end + + context 'ci database not configured' do + before do + skip_if_multiple_databases_are_setup + end + + it 'returns the main base model' do + expect(helper.active_record_base(database: :ci)).to eq(ActiveRecord::Base) + end + end + + it 'raises ArgumentError for bad database argument' do + expect { helper.active_record_base(database: :non_existent) }.to raise_error(ArgumentError) + end + end + + describe '#table' do + it 'creates a class based on main base model' do + klass = helper.table(:projects) + expect(klass.connection_specification_name).to eq('ActiveRecord::Base') + end + + context 'ci database configured' do + before do + skip_if_multiple_databases_not_setup + end + + it 'create a class based on the CI base model' do + klass = helper.table(:ci_builds, database: :ci) + expect(klass.connection_specification_name).to eq('Ci::ApplicationRecord') + end + end + + context 'ci database not configured' do + before do + skip_if_multiple_databases_are_setup + end + + it 'creates a class based on main base model' do + klass = helper.table(:ci_builds, database: :ci) + expect(klass.connection_specification_name).to eq('ActiveRecord::Base') + end + end + end +end diff --git a/yarn.lock b/yarn.lock index a221ba6e91f..dbe4fcd7f70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -968,10 +968,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-2.11.0.tgz#06edc30c58a539b2cb60ef30d61ce470c0f57f2b" integrity sha512-IkiMrt3NY4IHonEgSHfv6JoJ+3McppZpt7XYgG6QtVtiCHFuwbkTH+gLMeZHG127AjtuHr54wS/cYp4MSNZZ4Q== -"@gitlab/ui@39.6.0": - version "39.6.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-39.6.0.tgz#d39ee45a6b629498a60af9683ecc9c2d50486c13" - integrity sha512-RQBD4r8ii9xZ3Hwxfn3RKZkBWfberffX93MDWlxujD43DHJ0aE0Uhw/fqfE41i/uEUNqhhNVRtl77XLSlYP/iw== +"@gitlab/ui@40.0.0": + version "40.0.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-40.0.0.tgz#5a48a2c1ad509317ff0fde07070bd2070c82bd1a" + integrity sha512-LLnhju89i3usX66lSWQod222EFQjkBkiHG6frrDotW2PnCE6wB/xjlXumwpi93zfdyvIHS12cWPZwNa3ayiLfQ== dependencies: "@popperjs/core" "^2.11.2" bootstrap-vue "2.20.1" |