diff options
40 files changed, 954 insertions, 125 deletions
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 8204731e67a..a4bb99c49ad 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -73,6 +73,9 @@ .if-security-merge-request: &if-security-merge-request if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' +.if-fork-merge-request: &if-fork-merge-request + if: '$CI_PROJECT_NAMESPACE !~ /^gitlab(-org)?($|\/)/ && $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS !~ /pipeline:run-all-rspec/' + .if-default-branch-schedule-2-hourly: &if-default-branch-schedule-2-hourly if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' @@ -731,6 +734,8 @@ .frontend:rules:jest: rules: + - <<: *if-fork-merge-request + when: never - <<: *if-merge-request-labels-run-all-jest - <<: *if-default-refs changes: *core-frontend-patterns @@ -747,11 +752,12 @@ .frontend:rules:jest:minimal: rules: + - <<: *if-fork-merge-request + changes: *code-backstage-patterns - !reference [".rails:rules:minimal-default-rules", rules] - <<: *if-merge-request-labels-run-all-jest when: never - - <<: *if-default-refs - changes: *core-frontend-patterns + - changes: *core-frontend-patterns when: never - <<: *if-merge-request changes: *ci-patterns @@ -882,11 +888,15 @@ .rails:rules:ee-and-foss-migration: rules: + - <<: *if-fork-merge-request + when: never - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request changes: *db-patterns - <<: *if-automated-merge-request @@ -899,8 +909,12 @@ .rails:rules:ee-and-foss-migration:minimal: rules: + - <<: *if-fork-merge-request + changes: *db-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request changes: *db-patterns when: never @@ -921,11 +935,15 @@ .rails:rules:ee-and-foss-unit: rules: + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - changes: *backend-patterns .rails:rules:ee-and-foss-unit:minimal: rules: + - <<: *if-fork-merge-request + changes: *backend-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] - <<: *if-merge-request @@ -933,11 +951,15 @@ .rails:rules:ee-and-foss-integration: rules: + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - changes: *backend-patterns .rails:rules:ee-and-foss-integration:minimal: rules: + - <<: *if-fork-merge-request + changes: *backend-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] - <<: *if-merge-request @@ -945,11 +967,15 @@ .rails:rules:ee-and-foss-system: rules: + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:system-default-rules", rules] - changes: *code-backstage-patterns .rails:rules:ee-and-foss-system:minimal: rules: + - <<: *if-fork-merge-request + changes: *code-backstage-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:system:minimal-default-rules", rules] @@ -976,6 +1002,8 @@ changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request changes: *db-patterns - <<: *if-automated-merge-request @@ -992,6 +1020,8 @@ when: never - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request changes: *db-patterns when: never @@ -1000,6 +1030,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - changes: *backend-patterns @@ -1007,6 +1039,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + changes: *backend-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] - <<: *if-merge-request @@ -1016,6 +1050,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - changes: *backend-patterns @@ -1023,6 +1059,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + changes: *backend-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:unit-integration:minimal-default-rules", rules] - <<: *if-merge-request @@ -1032,6 +1070,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:system-default-rules", rules] - changes: *code-backstage-patterns @@ -1039,6 +1079,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + changes: *code-backstage-patterns - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:system:minimal-default-rules", rules] @@ -1051,8 +1093,8 @@ changes: *core-backend-patterns - <<: *if-merge-request changes: *ci-patterns - - <<: *if-security-merge-request - changes: *db-patterns + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request-labels-as-if-foss changes: *db-patterns - <<: *if-automated-merge-request @@ -1068,6 +1110,8 @@ when: never - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules", rules] + # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request-labels-as-if-foss changes: *db-patterns when: never @@ -1076,6 +1120,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns @@ -1084,6 +1130,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules", rules] - <<: *if-merge-request-labels-as-if-foss @@ -1093,6 +1141,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:ee-and-foss-default-rules", rules] - <<: *if-merge-request-labels-as-if-foss changes: *backend-patterns @@ -1101,6 +1151,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:minimal-default-rules", rules] - !reference [".rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules", rules] - <<: *if-merge-request-labels-as-if-foss @@ -1110,6 +1162,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:system-default-rules", rules] - <<: *if-merge-request-labels-as-if-foss changes: *code-backstage-patterns @@ -1118,6 +1172,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-fork-merge-request + when: never - !reference [".rails:rules:minimal-default-rules", rules] - <<: *if-merge-request changes: *core-backend-patterns diff --git a/.rubocop_todo/style/open_struct_use.yml b/.rubocop_todo/style/open_struct_use.yml index 80792631739..80239770db0 100644 --- a/.rubocop_todo/style/open_struct_use.yml +++ b/.rubocop_todo/style/open_struct_use.yml @@ -26,8 +26,5 @@ Style/OpenStructUse: - spec/lib/gitlab/gitaly_client/diff_stitcher_spec.rb - spec/lib/gitlab/legacy_github_import/project_creator_spec.rb - spec/lib/gitlab/quick_actions/command_definition_spec.rb - - spec/services/projects/import_service_spec.rb - spec/services/system_note_service_spec.rb - - spec/support/helpers/import_spec_helper.rb - - spec/support/helpers/login_helpers.rb - spec/support/helpers/repo_helpers.rb @@ -183,7 +183,7 @@ gem 'rack', '~> 2.2.3' gem 'rack-timeout', '~> 0.5.1', require: 'rack/timeout/base' group :puma do - gem 'puma', '~> 5.5.2', require: false + gem 'puma', '~> 5.6.2', require: false gem 'puma_worker_killer', '~> 0.3.1', require: false gem 'sd_notify', '~> 0.1.0', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 74ccb9dad70..71fe578b921 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -941,7 +941,7 @@ GEM tty-markdown tty-prompt public_suffix (4.0.6) - puma (5.5.2) + puma (5.6.2) nio4r (~> 2.0) puma_worker_killer (0.3.1) get_process_mem (~> 0.2) @@ -1571,7 +1571,7 @@ DEPENDENCIES pry-byebug pry-rails (~> 0.3.9) pry-shell (~> 0.5.0) - puma (~> 5.5.2) + puma (~> 5.6.2) puma_worker_killer (~> 0.3.1) rack (~> 2.2.3) rack-attack (~> 6.3.0) diff --git a/app/assets/javascripts/invite_members/components/invite_members_modal.vue b/app/assets/javascripts/invite_members/components/invite_members_modal.vue index a9bf1ccf6bb..6c0fc5caf26 100644 --- a/app/assets/javascripts/invite_members/components/invite_members_modal.vue +++ b/app/assets/javascripts/invite_members/components/invite_members_modal.vue @@ -8,6 +8,7 @@ import { GlFormCheckboxGroup, } from '@gitlab/ui'; import { partition, isString, uniqueId } from 'lodash'; +import InviteModalBase from 'ee_else_ce/invite_members/components/invite_modal_base.vue'; import Api from '~/api'; import ExperimentTracking from '~/experimentation/experiment_tracking'; import { BV_SHOW_MODAL, BV_HIDE_MODAL } from '~/lib/utils/constants'; @@ -21,7 +22,6 @@ import { import eventHub from '../event_hub'; import { responseMessageFromSuccess } from '../utils/response_message_parser'; import ModalConfetti from './confetti.vue'; -import InviteModalBase from './invite_modal_base.vue'; import MembersTokenSelect from './members_token_select.vue'; export default { diff --git a/app/assets/stylesheets/utilities.scss b/app/assets/stylesheets/utilities.scss index bb085e3f5ce..8a4f9c32f9f 100644 --- a/app/assets/stylesheets/utilities.scss +++ b/app/assets/stylesheets/utilities.scss @@ -307,6 +307,12 @@ to @gitlab/ui by https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1709 } } +.gl-lg-grid-template-columns-4 { + @include media-breakpoint-up(lg) { + grid-template-columns: repeat(4, 1fr); + } +} + .gl-gap-6 { gap: $gl-spacing-scale-6; } diff --git a/app/controllers/concerns/integrations/params.rb b/app/controllers/concerns/integrations/params.rb index 945540d1f8c..80acb369cb2 100644 --- a/app/controllers/concerns/integrations/params.rb +++ b/app/controllers/concerns/integrations/params.rb @@ -30,6 +30,7 @@ module Integrations :datadog_site, :datadog_env, :datadog_service, + :datadog_tags, :default_irc_uri, :device, :disable_diffs, diff --git a/app/helpers/invite_members_helper.rb b/app/helpers/invite_members_helper.rb index 884386ab8ac..1f225e9c0e5 100644 --- a/app/helpers/invite_members_helper.rb +++ b/app/helpers/invite_members_helper.rb @@ -86,3 +86,5 @@ module InviteMembersHelper projects.map { |project| { id: project.id, title: project.title } } end end + +InviteMembersHelper.prepend_mod_with('InviteMembersHelper') diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb index 6efede8d565..5b596c328d1 100644 --- a/app/helpers/search_helper.rb +++ b/app/helpers/search_helper.rb @@ -164,6 +164,23 @@ module SearchHelper options end + # search_context exposes a bit too much data to the frontend, this controls what data we share and when. + def header_search_context + {}.tap do |hash| + hash[:group] = { id: search_context.group.id, name: search_context.group.name } if search_context.for_group? + hash[:group_metadata] = search_context.group_metadata if search_context.for_group? + + hash[:project] = { id: search_context.project.id, name: search_context.project.name } if search_context.for_project? + hash[:project_metadata] = search_context.project_metadata if search_context.for_project? + + hash[:scope] = search_context.scope if search_context.for_project? || search_context.for_group? + hash[:code_search] = search_context.code_search? if search_context.for_project? || search_context.for_group? + + hash[:ref] = search_context.ref if can?(current_user, :download_code, search_context.project) + hash[:for_snippets] = search_context.for_snippets? + end + end + private # Autocomplete results for various settings pages diff --git a/app/models/concerns/has_environment_scope.rb b/app/models/concerns/has_environment_scope.rb index 9553abe4dd3..c01996c0c4c 100644 --- a/app/models/concerns/has_environment_scope.rb +++ b/app/models/concerns/has_environment_scope.rb @@ -70,6 +70,14 @@ module HasEnvironmentScope relation end + + scope :for_environment, ->(environment) do + if environment + on_environment(environment) + else + where(environment_scope: '*') + end + end end def environment_scope=(new_environment_scope) diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index b86f0aaa7ef..bb0fb6b9079 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -13,7 +13,11 @@ module Integrations pipeline job ].freeze - prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env + TAG_KEY_VALUE_RE = %r{\A [\w-]+ : .*\S.* \z}x.freeze + + prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env, :datadog_tags + + before_validation :strip_properties with_options if: :activated? do validates :api_key, presence: true, format: { with: /\A\w+\z/ } @@ -21,6 +25,7 @@ module Integrations validates :api_url, public_url: { allow_blank: true } validates :datadog_site, presence: true, unless: -> (obj) { obj.api_url.present? } validates :api_url, presence: true, unless: -> (obj) { obj.datadog_site.present? } + validate :datadog_tags_are_valid end def initialize_properties @@ -140,6 +145,20 @@ module Integrations linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, linkClose: '</a>'.html_safe } + }, + { + type: 'textarea', + name: 'datadog_tags', + title: s_('DatadogIntegration|Tags'), + placeholder: "tag:value\nanother_tag:value", + help: ERB::Util.html_escape( + s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } } ] @@ -153,7 +172,8 @@ module Integrations query = { "dd-api-key" => api_key, service: datadog_service.presence, - env: datadog_env.presence + env: datadog_env.presence, + tags: datadog_tags_query_param.presence }.compact url.query = query.to_query url.to_s @@ -193,5 +213,35 @@ module Integrations data end + + def strip_properties + datadog_service.strip! if datadog_service && !datadog_service.frozen? + datadog_env.strip! if datadog_env && !datadog_env.frozen? + datadog_tags.strip! if datadog_tags && !datadog_tags.frozen? + end + + def datadog_tags_are_valid + return unless datadog_tags + + unless datadog_tags.split("\n").select(&:present?).all? { _1 =~ TAG_KEY_VALUE_RE } + errors.add(:datadog_tags, s_("DatadogIntegration|have an invalid format")) + end + end + + def datadog_tags_query_param + return unless datadog_tags + + datadog_tags.split("\n").filter_map do |tag| + tag.strip! + + next if tag.blank? + + if tag.include?(',') + "\"#{tag}\"" + else + tag + end + end.join(',') + end end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 573c5c4d8f8..84518fd6b0e 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -140,8 +140,7 @@ module Auth type: type, name: path.to_s, actions: authorized_actions, - migration_eligible: self.class.migration_eligible(project: requested_project), - cdn_redirect: cdn_redirect + migration_eligible: self.class.migration_eligible(project: requested_project) }.compact end @@ -176,13 +175,6 @@ module Auth false end - # This is used to determine whether blob download requests using a given JWT token should be redirected to Google - # Cloud CDN or not. The intent is to enable a percentage of time rollout for this new feature on the Container - # Registry side. See https://gitlab.com/gitlab-org/gitlab/-/issues/349417 for more details. - def cdn_redirect - Feature.enabled?(:container_registry_cdn_redirect) || nil - end - ## # Because we do not have two way communication with registry yet, # we create a container repository image resource when push to the diff --git a/app/views/layouts/header/_default.html.haml b/app/views/layouts/header/_default.html.haml index 990f40d6aa2..871d1213c0e 100644 --- a/app/views/layouts/header/_default.html.haml +++ b/app/views/layouts/header/_default.html.haml @@ -41,7 +41,7 @@ %li.nav-item.d-none.d-lg-block.m-auto - unless current_controller?(:search) - if Feature.enabled?(:new_header_search) - #js-header-search.header-search{ data: { 'search-context' => search_context.to_json, + #js-header-search.header-search{ data: { 'search-context' => header_search_context.to_json, 'search-path' => search_path, 'issues-path' => issues_dashboard_path, 'mr-path' => merge_requests_dashboard_path, diff --git a/config/feature_flags/development/ci_variables_builder_memoize_secret_variables.yml b/config/feature_flags/development/ci_variables_builder_memoize_secret_variables.yml new file mode 100644 index 00000000000..a35b06f9907 --- /dev/null +++ b/config/feature_flags/development/ci_variables_builder_memoize_secret_variables.yml @@ -0,0 +1,8 @@ +--- +name: ci_variables_builder_memoize_secret_variables +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79850 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351995 +milestone: '14.8' +type: development +group: group::pipeline execution +default_enabled: false diff --git a/config/feature_flags/development/container_registry_cdn_redirect.yml b/config/feature_flags/development/overage_members_modal.yml index 5cc2bf7a342..16810f1acd2 100644 --- a/config/feature_flags/development/container_registry_cdn_redirect.yml +++ b/config/feature_flags/development/overage_members_modal.yml @@ -1,8 +1,8 @@ --- -name: container_registry_cdn_redirect -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77705 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/349717 -milestone: '14.7' +name: overage_members_modal +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79644/ +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/350265 +milestone: '14.8' type: development -group: group::package +group: group::purchase default_enabled: false diff --git a/db/migrate/20220208170445_add_not_valid_foreign_key_to_ci_builds_runner_id.rb b/db/migrate/20220208170445_add_not_valid_foreign_key_to_ci_builds_runner_id.rb new file mode 100644 index 00000000000..9b2ba17e068 --- /dev/null +++ b/db/migrate/20220208170445_add_not_valid_foreign_key_to_ci_builds_runner_id.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddNotValidForeignKeyToCiBuildsRunnerId < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :ci_builds, :ci_runners, column: :runner_id, on_delete: :nullify, validate: false + end + + def down + with_lock_retries do + remove_foreign_key_if_exists :ci_builds, column: :runner_id + end + end +end diff --git a/db/schema_migrations/20220208170445 b/db/schema_migrations/20220208170445 new file mode 100644 index 00000000000..3a486a586a8 --- /dev/null +++ b/db/schema_migrations/20220208170445 @@ -0,0 +1 @@ +e00dd618ca393596f3ff05b44b1a9a36183729a864a5cf4b8f1a262dfcdb932b
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 04cd900b364..763a4cf51be 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -30140,6 +30140,9 @@ ALTER TABLE ONLY ci_builds_metadata ALTER TABLE ONLY gitlab_subscriptions ADD CONSTRAINT fk_e2595d00a1 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_builds + ADD CONSTRAINT fk_e4ef9c2f27 FOREIGN KEY (runner_id) REFERENCES ci_runners(id) ON DELETE SET NULL NOT VALID; + ALTER TABLE ONLY merge_requests ADD CONSTRAINT fk_e719a85f8a FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; diff --git a/doc/api/integrations.md b/doc/api/integrations.md index d409cd47aae..90bb26ffd3d 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -322,6 +322,7 @@ Parameters: | `datadog_env` | string | false | For self-managed deployments, set the env% tag for all the data sent to Datadog. | | `datadog_service` | string | false | Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments | | `datadog_site` | string | false | The Datadog site to send data to. To send data to the EU site, use `datadoghq.eu` | +| `datadog_tags` | string | false | Custom tags in Datadog. Specify one tag per line in the format: `key:value\nkey2:value2` ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79665) in GitLab 14.8.) | <!-- | `archive_trace_events` | boolean | false | When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346339) in GitLab 14.7) | --> <!-- TODO: uncomment the archive_trace_events field once :datadog_integration_logs_collection is rolled out. Rollout issue: https://gitlab.com/gitlab-org/gitlab/-/issues/346339 --> diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 7011b3c6ef1..f3a4f47eb22 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -90,6 +90,13 @@ In addition, there are a few circumstances where we would always run the full Je - when any vendored JavaScript file is changed (i.e. `vendor/assets/javascripts/**/*`) - when any backend file is changed ([see the patterns list for details](https://gitlab.com/gitlab-org/gitlab/-/blob/3616946936c1adbd9e754c1bd06f86ba670796d8/.gitlab/ci/rules.gitlab-ci.yml#L205-216)) +### Fork pipelines + +We only run the minimal RSpec & Jest jobs for fork pipelines unless the `pipeline:run-all-rspec` +label is set on the MR. The goal is to reduce the CI minutes consumed by fork pipelines. + +See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1170). + ## Fail-fast job in merge request pipelines To provide faster feedback when a merge request breaks existing tests, we are experimenting with a @@ -176,6 +183,8 @@ Tests that are [known to be flaky](testing_guide/flaky_tests.md#automatic-retrie skipped unless the `$SKIP_FLAKY_TESTS_AUTOMATICALLY` variable is set to `false` or if the `~"pipeline:run-flaky-tests"` label is set on the MR. +See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1069). + #### Automatic retry of failing tests in a separate process When the `$RETRY_FAILED_TESTS_IN_NEW_PROCESS` variable is set to `true`, RSpec tests that failed are automatically retried once in a separate @@ -183,6 +192,8 @@ RSpec process. The goal is to get rid of most side-effects from previous tests t We keep track of retried tests in the `$RETRIED_TESTS_REPORT_FILE` file saved as artifact by the `rspec:flaky-tests-report` job. +See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148). + ### Monitoring The GitLab test suite is [monitored](performance.md#rspec-profiling) for the `main` branch, and any branch diff --git a/doc/integration/datadog.md b/doc/integration/datadog.md index 7f74786314f..a9be7754cb9 100644 --- a/doc/integration/datadog.md +++ b/doc/integration/datadog.md @@ -42,6 +42,8 @@ project, group, or instance level: 1. Optional. If you use groups of GitLab instances (such as staging and production environments), provide an **Env** name. This value is attached to each span the integration generates. +1. Optional. To define any custom tags for all spans at which the integration is being configured, + enter one tag per line in **Tags**. Each line must be in the format `key:value`. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79665) in GitLab 14.8.) 1. Optional. Select **Test settings** to test your integration. 1. Select **Save changes**. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 7ffb3181632..a169b78a193 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -767,6 +767,87 @@ Here's an example dependency scanning report: } ``` +### CycloneDX reports + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350509) in GitLab 14.8 in [Beta](../../../policy/alpha-beta-support.md#beta-features). + +In addition to the [JSON report file](#reports-json-format), the [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) +Dependency Scanning tool outputs a [CycloneDX](https://cyclonedx.org/) report for +each supported lock or build file it detects. These CycloneDX reports are named +`cyclonedx-<package-type>-<package-manager>.json`, and are saved in the same directory +as the detected lock or build files. + +For example, if your project has the following structure: + +```plaintext +. +├── ruby-project/ +│ └── Gemfile.lock +├── ruby-project-2/ +│ └── Gemfile.lock +├── php-project/ +│ └── composer.lock +└── go-project/ + └── go.sum +``` + +Then the Gemnasium scanner generates the following CycloneDX reports: + +```plaintext +. +├── ruby-project/ +│ ├── Gemfile.lock +│ └── cyclonedx-gem-bundler.json +├── ruby-project-2/ +│ ├── Gemfile.lock +│ └── cyclonedx-gem-bundler.json +├── php-project/ +│ ├── composer.lock +│ └── cyclonedx-packagist-composer.json +└── go-project/ + ├── go.sum + └── cyclonedx-go-go.json +``` + +The CycloneDX reports can be downloaded [the same way as other job artifacts](../../../ci/pipelines/job_artifacts.md#download-job-artifacts). + +### Merging multiple CycloneDX Reports + +You can use a CI/CD job to merge multiple CycloneDX Reports into a single report. +For example: + +```yaml +stages: + - test + - merge-cyclonedx-reports + +include: + - template: Security/Dependency-Scanning.gitlab-ci.yml + +merge cyclonedx reports: + stage: merge-cyclonedx-reports + image: alpine:latest + script: + - wget https://github.com/CycloneDX/cyclonedx-cli/releases/download/v0.22.0/cyclonedx-linux-musl-x64 -O /usr/local/bin/cyclonedx-cli + - chmod 755 /usr/local/bin/cyclonedx-cli + - apk --update add --no-cache icu-dev libstdc++ + - find * -name "cyclonedx-*.json" -exec cyclonedx-cli merge --input-files {} --output-file cyclonedx-all.json + + artifacts: + paths: + - cyclonedx-all.json +``` + +GitLab uses [CycloneDX Properties](https://cyclonedx.org/use-cases/#properties--name-value-store) +to store implementation-specific details in the metadata of each CycloneDX report, +such as the location of build and lock files. If multiple CycloneDX reports are merged together, +this information is removed from the resulting merged file. + +NOTE: +CycloneDX reports are a [Beta](../../../policy/alpha-beta-support.md#beta-features) feature, +and the reports are subject to change during the beta period. Do not build integrations +that rely on the format of these reports staying consistent, as the format might change +before the feature is made generally available. + ## Versioning and release process Please check the [Release Process documentation](https://gitlab.com/gitlab-org/security-products/release/blob/master/docs/release_process.md). diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 72b16a23dd6..86dedc12fca 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -346,7 +346,13 @@ module API required: false, name: :datadog_env, type: String, - desc: 'For self-managed deployments, set the env tag for all the data sent to Datadog. How do I use tags?' + desc: 'For self-managed deployments, set the env tag for all the data sent to Datadog' + }, + { + required: false, + name: :datadog_tags, + type: String, + desc: 'Custom tags in Datadog. Specify one tag per line in the format: "key:value\nkey2:value2"' } ], 'discord' => [ diff --git a/lib/feature.rb b/lib/feature.rb index 12b4ef07dd6..47fee23c7ea 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -245,11 +245,11 @@ class Feature end def gate_specified? - %i(user project group feature_group).any? { |key| params.key?(key) } + %i(user project group feature_group namespace).any? { |key| params.key?(key) } end def targets - [feature_group, user, project, group].compact + [feature_group, user, project, group, namespace].compact end private @@ -279,6 +279,13 @@ class Feature Group.find_by_full_path(params[:group]) end + + def namespace + return unless params.key?(:namespace) + + # We are interested in Group or UserNamespace + Namespace.without_project_namespaces.find_by_full_path(params[:namespace]) + end end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 90b84803cff..9ef6e7f5fa9 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -9,6 +9,7 @@ module Gitlab def initialize(pipeline) @pipeline = pipeline @instance_variables_builder = Builder::Instance.new + @project_variables_builder = Builder::Project.new(project) end def scoped_variables(job, environment:, dependencies:) @@ -77,13 +78,18 @@ module Gitlab end def secret_project_variables(environment:, ref:) - project.ci_variables_for(ref: ref, environment: environment) + if memoize_secret_variables? + memoized_secret_project_variables(environment: environment) + else + project.ci_variables_for(ref: ref, environment: environment) + end end private attr_reader :pipeline attr_reader :instance_variables_builder + attr_reader :project_variables_builder delegate :project, to: :pipeline def predefined_variables(job) @@ -104,6 +110,15 @@ module Gitlab end end + def memoized_secret_project_variables(environment:) + strong_memoize_with(:secret_project_variables, environment) do + project_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) + end + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) @@ -115,6 +130,24 @@ module Gitlab project.protected_for?(pipeline.jobs_git_ref) end end + + def memoize_secret_variables? + strong_memoize(:memoize_secret_variables) do + ::Feature.enabled?(:ci_variables_builder_memoize_secret_variables, + project, + default_enabled: :yaml) + end + end + + def strong_memoize_with(name, *args) + container = strong_memoize(name) { {} } + + if container.key?(args) + container[args] + else + container[args] = yield + end + end end end end diff --git a/lib/gitlab/ci/variables/builder/project.rb b/lib/gitlab/ci/variables/builder/project.rb new file mode 100644 index 00000000000..832e68ea6a2 --- /dev/null +++ b/lib/gitlab/ci/variables/builder/project.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Project + include Gitlab::Utils::StrongMemoize + + def initialize(project) + @project = project + end + + def secret_variables(environment:, protected_ref: false) + variables = @project.variables + variables = variables.unprotected unless protected_ref + variables = variables.for_environment(environment) + + Gitlab::Ci::Variables::Collection.new(variables) + end + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3f787075cf4..0cc286c4ba6 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -11399,6 +11399,9 @@ msgstr "" msgid "DatadogIntegration|API URL" msgstr "" +msgid "DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}" +msgstr "" + msgid "DatadogIntegration|Environment" msgstr "" @@ -11417,12 +11420,18 @@ msgstr "" msgid "DatadogIntegration|Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments." msgstr "" +msgid "DatadogIntegration|Tags" +msgstr "" + msgid "DatadogIntegration|The Datadog site to send data to. To send data to the EU site, use %{codeOpen}datadoghq.eu%{codeClose}." msgstr "" msgid "DatadogIntegration|Trace your GitLab pipelines with Datadog." msgstr "" +msgid "DatadogIntegration|have an invalid format" +msgstr "" + msgid "Datasource name not found" msgstr "" @@ -18629,12 +18638,18 @@ msgstr "" msgid "InProductMarketing|Access advanced features, build more efficiently, strengthen security and compliance." msgstr "" +msgid "InProductMarketing|Access advanced features." +msgstr "" + msgid "InProductMarketing|Actually, GitLab makes the team work (better)" msgstr "" msgid "InProductMarketing|And finally %{deploy_link} a Python application." msgstr "" +msgid "InProductMarketing|And many more..." +msgstr "" + msgid "InProductMarketing|Are your runners ready?" msgstr "" @@ -18656,15 +18671,24 @@ msgstr "" msgid "InProductMarketing|Break down silos to coordinate seamlessly across development, operations, and security with a consistent experience across the development lifecycle." msgstr "" +msgid "InProductMarketing|Burn up/down charts" +msgstr "" + msgid "InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process." msgstr "" msgid "InProductMarketing|Click on the number below that corresponds with your answer — 1 being very difficult, 5 being very easy." msgstr "" +msgid "InProductMarketing|Code owners" +msgstr "" + msgid "InProductMarketing|Code owners and required merge approvals are part of the paid tiers of GitLab. You can start a free 30-day trial of GitLab Ultimate and enable these features in less than 5 minutes with no credit card required." msgstr "" +msgid "InProductMarketing|Code review analytics" +msgstr "" + msgid "InProductMarketing|Collaboration across stages in GitLab" msgstr "" @@ -18680,12 +18704,21 @@ msgstr "" msgid "InProductMarketing|Create a project in GitLab in 5 minutes" msgstr "" +msgid "InProductMarketing|Create well-defined workflows by using scoped labels on issues, merge requests, and epics. Labels with the same scope cannot be used together, which prevents conflicts." +msgstr "" + msgid "InProductMarketing|Create your first project!" msgstr "" +msgid "InProductMarketing|Define who owns specific files or directories, so the right reviewers are suggested when a merge request introduces changes to those files." +msgstr "" + msgid "InProductMarketing|Deliver Better Products Faster" msgstr "" +msgid "InProductMarketing|Dependency scanning" +msgstr "" + msgid "InProductMarketing|Did you know teams that use GitLab are far more efficient?" msgstr "" @@ -18707,9 +18740,15 @@ msgstr "" msgid "InProductMarketing|Do you have a teammate who would be perfect for this task?" msgstr "" +msgid "InProductMarketing|Dynamic application security testing" +msgstr "" + msgid "InProductMarketing|Easy" msgstr "" +msgid "InProductMarketing|Epics" +msgstr "" + msgid "InProductMarketing|Expand your DevOps journey with a free GitLab trial" msgstr "" @@ -18731,9 +18770,15 @@ msgstr "" msgid "InProductMarketing|Feel the need for speed?" msgstr "" +msgid "InProductMarketing|Find and fix bottlenecks in your code review process by understanding how long open merge requests have been in review." +msgstr "" + msgid "InProductMarketing|Find out how your teams are really doing" msgstr "" +msgid "InProductMarketing|Find out if your external libraries are safe. Run dependency scanning jobs that check for known vulnerabilities in your external libraries." +msgstr "" + msgid "InProductMarketing|Follow our steps" msgstr "" @@ -18863,24 +18908,36 @@ msgstr "" msgid "InProductMarketing|It's also possible to simply %{external_repo_link} in order to take advantage of GitLab's CI/CD." msgstr "" +msgid "InProductMarketing|Keep your code quality high by defining who should approve merge requests and how many approvals are required." +msgstr "" + msgid "InProductMarketing|Launch GitLab CI/CD in 20 minutes or less" msgstr "" msgid "InProductMarketing|Lower cost of development" msgstr "" +msgid "InProductMarketing|Make it easier to collaborate on high-level ideas by grouping related issues in an epic." +msgstr "" + msgid "InProductMarketing|Making the switch? It's easier than you think to import your projects into GitLab. Move %{github_link}, or import something %{bitbucket_link}." msgstr "" msgid "InProductMarketing|Master the art of importing!" msgstr "" +msgid "InProductMarketing|Merge request approval rule" +msgstr "" + msgid "InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}" msgstr "" msgid "InProductMarketing|Multiple owners, confusing workstreams? We've got you covered" msgstr "" +msgid "InProductMarketing|Multiple required approvers" +msgstr "" + msgid "InProductMarketing|Need an alternative to importing?" msgstr "" @@ -18893,12 +18950,24 @@ msgstr "" msgid "InProductMarketing|Our tool brings all the things together" msgstr "" +msgid "InProductMarketing|Protect your web application by using DAST to examine for vulnerabilities in deployed environments." +msgstr "" + msgid "InProductMarketing|Rapid development, simplified" msgstr "" msgid "InProductMarketing|Reduce Security & Compliance Risk" msgstr "" +msgid "InProductMarketing|Require multiple approvers on a merge request, so you know it's in good shape before it's merged." +msgstr "" + +msgid "InProductMarketing|Roadmaps" +msgstr "" + +msgid "InProductMarketing|Scoped labels" +msgstr "" + msgid "InProductMarketing|Security that's integrated into your development lifecycle" msgstr "" @@ -18986,6 +19055,9 @@ msgstr "" msgid "InProductMarketing|To understand and get the most out of GitLab, start at the beginning and %{project_link}. In GitLab, repositories are part of a project, so after you've created your project you can go ahead and %{repo_link}." msgstr "" +msgid "InProductMarketing|Track completed issues in a chart, so you can see how a milestone is progressing at a glance." +msgstr "" + msgid "InProductMarketing|Try GitLab Ultimate for free" msgstr "" @@ -19022,6 +19094,9 @@ msgstr "" msgid "InProductMarketing|Very easy" msgstr "" +msgid "InProductMarketing|Visualize your epics and milestones in a timeline." +msgstr "" + msgid "InProductMarketing|Want to host GitLab on your servers?" msgstr "" @@ -22613,6 +22688,19 @@ msgstr "" msgid "Members of a group may only view projects they have permission to access" msgstr "" +msgid "MembersOverage|If you continue, the %{groupName} group will have %{quantity} seat in use and will be billed for the overage." +msgid_plural "MembersOverage|If you continue, the %{groupName} group will have %{quantity} seats in use and will be billed for the overage." +msgstr[0] "" +msgstr[1] "" + +msgid "MembersOverage|You are about to incur additional charges" +msgstr "" + +msgid "MembersOverage|Your subscription includes %d seat." +msgid_plural "MembersOverage|Your subscription includes %d seats." +msgstr[0] "" +msgstr[1] "" + msgid "Membership" msgstr "" diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 9bd6691bdb2..2608a13a399 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -26,7 +26,7 @@ RSpec.describe 'Database schema' do boards: %w[milestone_id iteration_id], chat_names: %w[chat_id team_id user_id], chat_teams: %w[team_id], - ci_builds: %w[erased_by_id runner_id trigger_request_id], + ci_builds: %w[erased_by_id trigger_request_id], ci_namespace_monthly_usages: %w[namespace_id], ci_runner_projects: %w[runner_id], ci_trigger_requests: %w[commit_id], diff --git a/spec/helpers/search_helper_spec.rb b/spec/helpers/search_helper_spec.rb index 40cfdafc9ac..78cc1dcee01 100644 --- a/spec/helpers/search_helper_spec.rb +++ b/spec/helpers/search_helper_spec.rb @@ -658,4 +658,152 @@ RSpec.describe SearchHelper do expect(search_sort_options).to eq(mock_created_sort) end end + + describe '#header_search_context' do + let(:user) { create(:user) } + let(:can_download) { false } + + let(:for_group) { false } + let(:group) { nil } + let(:group_metadata) { nil } + + let(:for_project) { false } + let(:project) { nil } + let(:project_metadata) { nil } + + let(:scope) { nil } + let(:code_search) { false } + let(:ref) { nil } + let(:for_snippets) { false } + + let(:search_context) do + instance_double(Gitlab::SearchContext, + group: group, + group_metadata: group_metadata, + project: project, + project_metadata: project_metadata, + scope: scope, + ref: ref) + end + + before do + allow(self).to receive(:search_context).and_return(search_context) + allow(self).to receive(:current_user).and_return(user) + allow(self).to receive(:can?).and_return(can_download) + + allow(search_context).to receive(:for_group?).and_return(for_group) + allow(search_context).to receive(:for_project?).and_return(for_project) + + allow(search_context).to receive(:code_search?).and_return(code_search) + allow(search_context).to receive(:for_snippets?).and_return(for_snippets) + end + + context 'group data' do + let(:group) { create(:group) } + let(:group_metadata) { { group_path: group.path, issues_path: "/issues" } } + let(:scope) { 'issues' } + let(:code_search) { true } + + context 'when for_group? is true' do + let(:for_group) { true } + + it 'adds the :group and :group_metadata correctly to hash' do + expect(header_search_context[:group]).to eq({ id: group.id, name: group.name }) + expect(header_search_context[:group_metadata]).to eq(group_metadata) + end + + it 'adds scope and code_search? correctly to hash' do + expect(header_search_context[:scope]).to eq(scope) + expect(header_search_context[:code_search]).to eq(code_search) + end + end + + context 'when for_group? is false' do + let(:for_group) { false } + + it 'does not add the :group and :group_metadata to hash' do + expect(header_search_context[:group]).to eq(nil) + expect(header_search_context[:group_metadata]).to eq(nil) + end + + it 'does not add scope and code_search? to hash' do + expect(header_search_context[:scope]).to eq(nil) + expect(header_search_context[:code_search]).to eq(nil) + end + end + end + + context 'project data' do + let(:project) { create(:project) } + let(:project_metadata) { { project_path: project.path, issues_path: "/issues" } } + let(:scope) { 'issues' } + let(:code_search) { true } + + context 'when for_project? is true' do + let(:for_project) { true } + + it 'adds the :project and :project_metadata correctly to hash' do + expect(header_search_context[:project]).to eq({ id: project.id, name: project.name }) + expect(header_search_context[:project_metadata]).to eq(project_metadata) + end + + it 'adds scope and code_search? correctly to hash' do + expect(header_search_context[:scope]).to eq(scope) + expect(header_search_context[:code_search]).to eq(code_search) + end + end + + context 'when for_project? is false' do + let(:for_project) { false } + + it 'does not add the :project and :project_metadata to hash' do + expect(header_search_context[:project]).to eq(nil) + expect(header_search_context[:project_metadata]).to eq(nil) + end + + it 'does not add scope and code_search? to hash' do + expect(header_search_context[:scope]).to eq(nil) + expect(header_search_context[:code_search]).to eq(nil) + end + end + end + + context 'ref data' do + let(:ref) { 'test-branch' } + + context 'when user can? download project data' do + let(:can_download) { true } + + it 'adds the :ref correctly to hash' do + expect(header_search_context[:ref]).to eq(ref) + end + end + + context 'when user cannot download project data' do + let(:can_download) { false } + + it 'does not add the :ref to hash' do + expect(header_search_context[:ref]).to eq(nil) + end + end + end + + context 'snippets' do + context 'when for_snippets? is true' do + let(:for_snippets) { true } + + it 'adds :for_snippets correctly to hash' do + expect(header_search_context[:for_snippets]).to eq(for_snippets) + end + end + + context 'when for_snippets? is false' do + let(:for_snippets) { false } + + it 'adds :for_snippets correctly to hash' do + expect(header_search_context[:for_snippets]).to eq(for_snippets) + end + end + end + end end diff --git a/spec/lib/gitlab/ci/variables/builder/project_spec.rb b/spec/lib/gitlab/ci/variables/builder/project_spec.rb new file mode 100644 index 00000000000..b64b6ea98e2 --- /dev/null +++ b/spec/lib/gitlab/ci/variables/builder/project_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Variables::Builder::Project do + let_it_be(:project) { create(:project, :repository) } + + let(:builder) { described_class.new(project) } + + describe '#secret_variables' do + let(:environment) { '*' } + let(:protected_ref) { false } + + let_it_be(:variable) do + create(:ci_variable, + value: 'secret', + project: project) + end + + let_it_be(:protected_variable) do + create(:ci_variable, :protected, + value: 'protected', + project: project) + end + + let(:variable_item) { item(variable) } + let(:protected_variable_item) { item(protected_variable) } + + subject do + builder.secret_variables( + environment: environment, + protected_ref: protected_ref) + end + + context 'when the ref is protected' do + let(:protected_ref) { true } + + it 'contains all the variables' do + is_expected.to contain_exactly(variable_item, protected_variable_item) + end + end + + context 'when the ref is not protected' do + let(:protected_ref) { false } + + it 'contains only the unprotected variables' do + is_expected.to contain_exactly(variable_item) + end + end + + context 'when environment name is specified' do + let(:environment) { 'review/name' } + + before do + Ci::Variable.update_all(environment_scope: environment_scope) + end + + context 'when environment scope is exactly matched' do + let(:environment_scope) { 'review/name' } + + it { is_expected.to contain_exactly(variable_item) } + end + + context 'when environment scope is matched by wildcard' do + let(:environment_scope) { 'review/*' } + + it { is_expected.to contain_exactly(variable_item) } + end + + context 'when environment scope does not match' do + let(:environment_scope) { 'review/*/special' } + + it { is_expected.not_to contain_exactly(variable_item) } + end + + context 'when environment scope has _' do + let(:environment_scope) { '*_*' } + + it 'does not treat it as wildcard' do + is_expected.not_to contain_exactly(variable_item) + end + end + + context 'when environment name contains underscore' do + let(:environment) { 'foo_bar/test' } + let(:environment_scope) { 'foo_bar/*' } + + it 'matches literally for _' do + is_expected.to contain_exactly(variable_item) + end + end + + # The environment name and scope cannot have % at the moment, + # but we're considering relaxing it and we should also make sure + # it doesn't break in case some data sneaked in somehow as we're + # not checking this integrity in database level. + context 'when environment scope has %' do + let(:environment_scope) { '*%*' } + + it 'does not treat it as wildcard' do + is_expected.not_to contain_exactly(variable_item) + end + end + + context 'when environment name contains a percent' do + let(:environment) { 'foo%bar/test' } + let(:environment_scope) { 'foo%bar/*' } + + it 'matches literally for _' do + is_expected.to contain_exactly(variable_item) + end + end + end + + context 'when variables with the same name have different environment scopes' do + let(:environment) { 'review/name' } + + let_it_be(:partially_matched_variable) do + create(:ci_variable, + key: variable.key, + value: 'partial', + environment_scope: 'review/*', + project: project) + end + + let_it_be(:perfectly_matched_variable) do + create(:ci_variable, + key: variable.key, + value: 'prefect', + environment_scope: 'review/name', + project: project) + end + + it 'puts variables matching environment scope more in the end' do + variables_collection = Gitlab::Ci::Variables::Collection.new([ + variable, + partially_matched_variable, + perfectly_matched_variable + ]).to_runner_variables + + expect(subject.to_runner_variables).to eq(variables_collection) + end + end + end + + def item(variable) + Gitlab::Ci::Variables::Collection::Item.fabricate(variable) + end +end diff --git a/spec/lib/gitlab/ci/variables/builder_spec.rb b/spec/lib/gitlab/ci/variables/builder_spec.rb index d2370b9d43c..6e144d62ac0 100644 --- a/spec/lib/gitlab/ci/variables/builder_spec.rb +++ b/spec/lib/gitlab/ci/variables/builder_spec.rb @@ -349,14 +349,93 @@ RSpec.describe Gitlab::Ci::Variables::Builder do end describe '#secret_project_variables' do - subject { builder.secret_project_variables(ref: job.git_ref, environment: job.expanded_environment_name) } - let_it_be(:protected_variable) { create(:ci_variable, protected: true, project: project) } let_it_be(:unprotected_variable) { create(:ci_variable, protected: false, project: project) } - let(:protected_variable_item) { protected_variable } - let(:unprotected_variable_item) { unprotected_variable } + let(:ref) { job.git_ref } + let(:environment) { job.expanded_environment_name } - include_examples "secret CI variables" + subject { builder.secret_project_variables(ref: ref, environment: environment) } + + context 'with ci_variables_builder_memoize_secret_variables disabled' do + before do + stub_feature_flags(ci_variables_builder_memoize_secret_variables: false) + end + + let(:protected_variable_item) { protected_variable } + let(:unprotected_variable_item) { unprotected_variable } + + include_examples "secret CI variables" + end + + context 'with ci_variables_builder_memoize_secret_variables enabled' do + before do + stub_feature_flags(ci_variables_builder_memoize_secret_variables: true) + end + + let(:protected_variable_item) { Gitlab::Ci::Variables::Collection::Item.fabricate(protected_variable) } + let(:unprotected_variable_item) { Gitlab::Ci::Variables::Collection::Item.fabricate(unprotected_variable) } + + include_examples "secret CI variables" + + context 'variables memoization' do + let_it_be(:scoped_variable) { create(:ci_variable, project: project, environment_scope: 'scoped') } + + let(:scoped_variable_item) { Gitlab::Ci::Variables::Collection::Item.fabricate(scoped_variable) } + + context 'with protected environments' do + it 'memoizes the result by environment' do + expect(pipeline.project) + .to receive(:protected_for?) + .with(pipeline.jobs_git_ref) + .once.and_return(true) + + expect_next_instance_of(described_class::Project) do |project_variables_builder| + expect(project_variables_builder) + .to receive(:secret_variables) + .with(environment: 'production', protected_ref: true) + .once + .and_call_original + end + + 2.times do + expect(builder.secret_project_variables(ref: ref, environment: 'production')) + .to contain_exactly(unprotected_variable_item, protected_variable_item) + end + end + end + + context 'with unprotected environments' do + it 'memoizes the result by environment' do + expect(pipeline.project) + .to receive(:protected_for?) + .with(pipeline.jobs_git_ref) + .once.and_return(false) + + expect_next_instance_of(described_class::Project) do |project_variables_builder| + expect(project_variables_builder) + .to receive(:secret_variables) + .with(environment: nil, protected_ref: false) + .once + .and_call_original + + expect(project_variables_builder) + .to receive(:secret_variables) + .with(environment: 'scoped', protected_ref: false) + .once + .and_call_original + end + + 2.times do + expect(builder.secret_project_variables(ref: 'other', environment: nil)) + .to contain_exactly(unprotected_variable_item) + + expect(builder.secret_project_variables(ref: 'other', environment: 'scoped')) + .to contain_exactly(unprotected_variable_item, scoped_variable_item) + end + end + end + end + end end end diff --git a/spec/models/concerns/has_environment_scope_spec.rb b/spec/models/concerns/has_environment_scope_spec.rb index 0cc997709c9..6e8394b6fa5 100644 --- a/spec/models/concerns/has_environment_scope_spec.rb +++ b/spec/models/concerns/has_environment_scope_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe HasEnvironmentScope do + let_it_be(:project) { create(:project) } + subject { build(:ci_variable) } it { is_expected.to allow_value('*').for(:environment_scope) } @@ -17,8 +19,6 @@ RSpec.describe HasEnvironmentScope do end describe '.on_environment' do - let(:project) { create(:project) } - it 'returns scoped objects' do variable1 = create(:ci_variable, project: project, environment_scope: '*') variable2 = create(:ci_variable, project: project, environment_scope: 'product/*') @@ -63,4 +63,32 @@ RSpec.describe HasEnvironmentScope do end end end + + describe '.for_environment' do + subject { project.variables.for_environment(environment) } + + let_it_be(:variable1) do + create(:ci_variable, project: project, environment_scope: '*') + end + + let_it_be(:variable2) do + create(:ci_variable, project: project, environment_scope: 'production/*') + end + + let_it_be(:variable3) do + create(:ci_variable, project: project, environment_scope: 'staging/*') + end + + context 'when the environment is present' do + let(:environment) { 'production/canary-1' } + + it { is_expected.to eq([variable1, variable2]) } + end + + context 'when the environment is nil' do + let(:environment) {} + + it { is_expected.to eq([variable1]) } + end + end end diff --git a/spec/models/integrations/datadog_spec.rb b/spec/models/integrations/datadog_spec.rb index 9856c53a390..cfc44b22a84 100644 --- a/spec/models/integrations/datadog_spec.rb +++ b/spec/models/integrations/datadog_spec.rb @@ -16,6 +16,7 @@ RSpec.describe Integrations::Datadog do let(:api_key) { SecureRandom.hex(32) } let(:dd_env) { 'ci' } let(:dd_service) { 'awesome-gitlab' } + let(:dd_tags) { '' } let(:expected_hook_url) { default_url + "?dd-api-key=#{api_key}&env=#{dd_env}&service=#{dd_service}" } @@ -27,7 +28,8 @@ RSpec.describe Integrations::Datadog do api_url: api_url, api_key: api_key, datadog_env: dd_env, - datadog_service: dd_service + datadog_service: dd_service, + datadog_tags: dd_tags ) end @@ -95,6 +97,20 @@ RSpec.describe Integrations::Datadog do it { is_expected.not_to allow_value('datadog hq.com').for(:datadog_site) } it { is_expected.not_to allow_value('example.com').for(:api_url) } end + + context 'with custom tags' do + it { is_expected.to allow_value('').for(:datadog_tags) } + it { is_expected.to allow_value('key:value').for(:datadog_tags) } + it { is_expected.to allow_value("key:value\nkey2:value2").for(:datadog_tags) } + it { is_expected.to allow_value("key:value\nkey2:value with spaces and 123?&$").for(:datadog_tags) } + it { is_expected.to allow_value("key:value\n\n\n\nkey2:value2\n").for(:datadog_tags) } + + it { is_expected.not_to allow_value('value').for(:datadog_tags) } + it { is_expected.not_to allow_value('key:').for(:datadog_tags) } + it { is_expected.not_to allow_value('key: ').for(:datadog_tags) } + it { is_expected.not_to allow_value(':value').for(:datadog_tags) } + it { is_expected.not_to allow_value("key:value\nINVALID").for(:datadog_tags) } + end end context 'when integration is not active' do @@ -134,9 +150,23 @@ RSpec.describe Integrations::Datadog do context 'without optional params' do let(:dd_service) { '' } let(:dd_env) { '' } + let(:dd_tags) { '' } it { is_expected.to eq(default_url + "?dd-api-key=#{api_key}") } end + + context 'with custom tags' do + let(:dd_tags) { "key:value\nkey2:value, 2" } + let(:escaped_tags) { CGI.escape("key:value,\"key2:value, 2\"") } + + it { is_expected.to eq(expected_hook_url + "&tags=#{escaped_tags}") } + + context 'and empty lines' do + let(:dd_tags) { "key:value\r\n\n\n\nkey2:value, 2\n" } + + it { is_expected.to eq(expected_hook_url + "&tags=#{escaped_tags}") } + end + end end describe '#test' do diff --git a/spec/requests/api/features_spec.rb b/spec/requests/api/features_spec.rb index 35dba93b766..a265f67115a 100644 --- a/spec/requests/api/features_spec.rb +++ b/spec/requests/api/features_spec.rb @@ -167,76 +167,85 @@ RSpec.describe API::Features, stub_feature_flags: false do end end + shared_examples 'does not enable the flag' do |actor_type, actor_path| + it 'returns the current state of the flag without changes' do + post api("/features/#{feature_name}", admin), params: { value: 'true', actor_type => actor_path } + + expect(response).to have_gitlab_http_status(:created) + expect(json_response).to match( + "name" => feature_name, + "state" => "off", + "gates" => [ + { "key" => "boolean", "value" => false } + ], + 'definition' => known_feature_flag_definition_hash + ) + end + end + + shared_examples 'enables the flag for the actor' do |actor_type| + it 'sets the feature gate' do + post api("/features/#{feature_name}", admin), params: { value: 'true', actor_type => actor.full_path } + + expect(response).to have_gitlab_http_status(:created) + expect(json_response).to match( + 'name' => feature_name, + 'state' => 'conditional', + 'gates' => [ + { 'key' => 'boolean', 'value' => false }, + { 'key' => 'actors', 'value' => ["#{actor.class}:#{actor.id}"] } + ], + 'definition' => known_feature_flag_definition_hash + ) + end + end + context 'when enabling for a project by path' do context 'when the project exists' do - let!(:project) { create(:project) } - - it 'sets the feature gate' do - post api("/features/#{feature_name}", admin), params: { value: 'true', project: project.full_path } - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to match( - 'name' => feature_name, - 'state' => 'conditional', - 'gates' => [ - { 'key' => 'boolean', 'value' => false }, - { 'key' => 'actors', 'value' => ["Project:#{project.id}"] } - ], - 'definition' => known_feature_flag_definition_hash - ) + it_behaves_like 'enables the flag for the actor', :project do + let(:actor) { create(:project) } end end context 'when the project does not exist' do - it 'sets no new values' do - post api("/features/#{feature_name}", admin), params: { value: 'true', project: 'mep/to/the/mep/mep' } - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to match( - "name" => feature_name, - "state" => "off", - "gates" => [ - { "key" => "boolean", "value" => false } - ], - 'definition' => known_feature_flag_definition_hash - ) - end + it_behaves_like 'does not enable the flag', :project, 'mep/to/the/mep/mep' end end context 'when enabling for a group by path' do context 'when the group exists' do - it 'sets the feature gate' do - group = create(:group) - - post api("/features/#{feature_name}", admin), params: { value: 'true', group: group.full_path } - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to match( - 'name' => feature_name, - 'state' => 'conditional', - 'gates' => [ - { 'key' => 'boolean', 'value' => false }, - { 'key' => 'actors', 'value' => ["Group:#{group.id}"] } - ], - 'definition' => known_feature_flag_definition_hash - ) + it_behaves_like 'enables the flag for the actor', :group do + let(:actor) { create(:group) } end end context 'when the group does not exist' do - it 'sets no new values and keeps the feature disabled' do - post api("/features/#{feature_name}", admin), params: { value: 'true', group: 'not/a/group' } - - expect(response).to have_gitlab_http_status(:created) - expect(json_response).to match( - "name" => feature_name, - "state" => "off", - "gates" => [ - { "key" => "boolean", "value" => false } - ], - 'definition' => known_feature_flag_definition_hash - ) + it_behaves_like 'does not enable the flag', :group, 'not/a/group' + end + end + + context 'when enabling for a namespace by path' do + context 'when the user namespace exists' do + it_behaves_like 'enables the flag for the actor', :namespace do + let(:actor) { create(:namespace) } + end + end + + context 'when the group namespace exists' do + it_behaves_like 'enables the flag for the actor', :namespace do + let(:actor) { create(:group) } + end + end + + context 'when the user namespace does not exist' do + it_behaves_like 'does not enable the flag', :namespace, 'not/a/group' + end + + context 'when a project namespace exists' do + let(:project_namespace) { create(:project_namespace) } + + it_behaves_like 'does not enable the flag', :namespace do + let(:actor_path) { project_namespace.full_path } end end end diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index 83f77780b80..00841de9ff4 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -145,28 +145,4 @@ RSpec.describe Auth::ContainerRegistryAuthenticationService do it_behaves_like 'an unmodified token' end end - - context 'CDN redirection' do - include_context 'container registry auth service context' - - let_it_be(:current_user) { create(:user) } - let_it_be(:project) { create(:project) } - let_it_be(:current_params) { { scopes: ["repository:#{project.full_path}:pull"] } } - - before do - project.add_developer(current_user) - end - - it_behaves_like 'a valid token' - it { expect(payload['access']).to include(include('cdn_redirect' => true)) } - - context 'when the feature flag is disabled' do - before do - stub_feature_flags(container_registry_cdn_redirect: false) - end - - it_behaves_like 'a valid token' - it { expect(payload['access']).not_to include(have_key('cdn_redirect')) } - end - end end diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 1d63f72ec38..ccfd119b55b 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -298,7 +298,7 @@ RSpec.describe Projects::ImportService do end def stub_github_omniauth_provider - provider = OpenStruct.new( + provider = ActiveSupport::InheritableOptions.new( 'name' => 'github', 'app_id' => 'asd123', 'app_secret' => 'asd123', diff --git a/spec/support/helpers/import_spec_helper.rb b/spec/support/helpers/import_spec_helper.rb index d8fb2ba08af..26b78acbc44 100644 --- a/spec/support/helpers/import_spec_helper.rb +++ b/spec/support/helpers/import_spec_helper.rb @@ -25,7 +25,7 @@ module ImportSpecHelper end def stub_omniauth_provider(name) - provider = OpenStruct.new( + provider = ActiveSupport::InheritableOptions.new( name: name, app_id: 'asd123', app_secret: 'asd123' diff --git a/spec/support/helpers/login_helpers.rb b/spec/support/helpers/login_helpers.rb index 4e0e8dd96ee..c0734bae375 100644 --- a/spec/support/helpers/login_helpers.rb +++ b/spec/support/helpers/login_helpers.rb @@ -178,7 +178,7 @@ module LoginHelpers end def mock_saml_config - OpenStruct.new(name: 'saml', label: 'saml', args: { + ActiveSupport::InheritableOptions.new(name: 'saml', label: 'saml', args: { assertion_consumer_service_url: 'https://localhost:3443/users/auth/saml/callback', idp_cert_fingerprint: '26:43:2C:47:AF:F0:6B:D0:07:9C:AD:A3:74:FE:5D:94:5F:4E:9E:52', idp_sso_target_url: 'https://idp.example.com/sso/saml', diff --git a/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb b/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb index 3d2b0433b21..3ea6658c0c1 100644 --- a/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb +++ b/spec/support/shared_contexts/features/integrations/integrations_shared_context.rb @@ -18,6 +18,8 @@ Integration.available_integration_names.each do |integration| hash.merge!(k => 'https://example.atlassian.net/wiki') elsif integration == 'datadog' && k == :datadog_site hash.merge!(k => 'datadoghq.com') + elsif integration == 'datadog' && k == :datadog_tags + hash.merge!(k => 'key:value') elsif integration == 'packagist' && k == :server hash.merge!(k => 'https://packagist.example.com') elsif k =~ /^(.*_url|url|webhook)/ diff --git a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb index c296332f38f..c808b9a5318 100644 --- a/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb +++ b/spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb @@ -71,7 +71,6 @@ end RSpec.shared_examples 'an accessible' do before do stub_feature_flags(container_registry_migration_phase1: false) - stub_feature_flags(container_registry_cdn_redirect: false) end let(:access) do @@ -164,7 +163,6 @@ RSpec.shared_examples 'a container registry auth service' do before do stub_feature_flags(container_registry_migration_phase1: false) - stub_feature_flags(container_registry_cdn_redirect: false) end describe '.full_access_token' do |