diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-22 18:10:52 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-22 18:10:52 +0300 |
commit | b52aefb5996cdd22dc969161d414244a59046e6e (patch) | |
tree | 9b1e3fe92ababfcc3110dca0380727a593220bd4 | |
parent | 4fc46d75644b28789e83c95ec4d1309498bb4ba3 (diff) |
Add latest changes from gitlab-org/gitlab@master
64 files changed, 1309 insertions, 132 deletions
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index dd615fe5e9d..51e23dce320 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -179,11 +179,6 @@ cache: - *ruby-coverage-gems-cache-push -# This cache should eventually be replaced by .ruby-gems-coverage-cache. -.coverage-cache: - cache: - - *ruby-gems-cache - .ruby-node-cache: cache: - *ruby-gems-cache diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 326d23be5a4..30563fee42c 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -68,19 +68,17 @@ update-ruby-gems-coverage-cache-push: - bundle_install_script # Used in: +# - rspec:coverage # - rspec:undercoverage -# - rspec:feature-flags -# - rspec:merge-auto-explain-logs -# -# TODO: Consider making rspec:coverage reuse this base job, or split into two base jobs. .coverage-base: extends: - .default-retry - # TODO: If applicable for all children jobs, delete/replace this cache with .ruby-gems-coverage-cache, as it is much smaller. - - .coverage-cache + - .ruby-gems-coverage-cache + variables: + BUNDLE_WITHOUT: "" # This is to override the variable defined in .gitlab-ci.yml + BUNDLE_ONLY: "coverage" before_script: - source scripts/utils.sh - - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" - bundle_install_script rspec migration pg14: @@ -463,8 +461,9 @@ rspec:artifact-collector ee: - !reference ['.rails:rules:ee-only-system', rules] rspec:coverage: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim-ruby-${RUBY_VERSION} extends: - - .ruby-gems-coverage-cache + - .coverage-base - .rails:rules:rspec-coverage stage: post-test needs: @@ -497,12 +496,6 @@ rspec:coverage: - job: memory-on-boot optional: true artifacts: false - variables: - BUNDLE_WITHOUT: "" # This is to override the variable defined in .gitlab-ci.yml - BUNDLE_ONLY: "coverage" - before_script: - - source scripts/utils.sh - - bundle_install_script script: - run_timed_command "bundle exec scripts/merge-simplecov" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' @@ -519,6 +512,7 @@ rspec:coverage: path: coverage/coverage.xml rspec:undercoverage: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim-ruby-${RUBY_VERSION} extends: - .coverage-base - .rails:rules:rspec-undercoverage @@ -539,7 +533,6 @@ rspec:undercoverage: rspec:feature-flags: extends: - - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test needs: @@ -547,11 +540,13 @@ rspec:feature-flags: - job: "haml-lint" - job: "haml-lint ee" optional: true + before_script: + - source scripts/utils.sh script: - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); + run_timed_command "scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); else - run_timed_command "bundle exec scripts/used-feature-flags"; + run_timed_command "scripts/used-feature-flags"; fi rspec:flaky-tests-report: @@ -572,7 +567,6 @@ rspec:flaky-tests-report: rspec:merge-auto-explain-logs: extends: - - .coverage-base - .rails:rules:rspec-merge-auto-explain-logs stage: post-test needs: !reference ["rspec:coverage", "needs"] diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index ded897d8d11..2531721be45 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -2248,6 +2248,8 @@ when: never - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request-labels-record-queries + - <<: *if-default-branch-refs + changes: *code-patterns .rails:rules:default-branch-schedule-nightly--code-backstage-default-rules: rules: diff --git a/app/assets/javascripts/sidebar/components/labels/labels_select_widget/dropdown_contents_labels_view.vue b/app/assets/javascripts/sidebar/components/labels/labels_select_widget/dropdown_contents_labels_view.vue index 19fe78aca87..d8d07b4b221 100644 --- a/app/assets/javascripts/sidebar/components/labels/labels_select_widget/dropdown_contents_labels_view.vue +++ b/app/assets/javascripts/sidebar/components/labels/labels_select_widget/dropdown_contents_labels_view.vue @@ -135,6 +135,16 @@ export default { this.handleLabelClick(this.visibleLabels[0]); } }, + handleFocus(event, index) { + if (index === 0 && event.target.classList.contains('is-focused')) { + event.target.classList.remove('is-focused'); + + // Focus next element (if available) as the first item was already focused. + if (event.target.parentNode?.nextElementSibling?.querySelector('button')) { + event.target.parentNode.nextElementSibling.querySelector('button').focus(); + } + } + }, }, }; </script> @@ -157,6 +167,7 @@ export default { :active="shouldHighlightFirstItem && index === 0" active-class="is-focused" data-testid="labels-list" + @focus.native.capture="handleFocus($event, index)" @click.native.capture.stop="handleLabelClick(label)" > <label-item :label="label" /> diff --git a/app/assets/javascripts/super_sidebar/components/nav_item.vue b/app/assets/javascripts/super_sidebar/components/nav_item.vue index 36803a885e7..227f73a02c4 100644 --- a/app/assets/javascripts/super_sidebar/components/nav_item.vue +++ b/app/assets/javascripts/super_sidebar/components/nav_item.vue @@ -162,6 +162,7 @@ export default { v-if="hasPill" size="sm" variant="neutral" + class="gl-bg-t-gray-a-08!" :class="{ 'nav-item-badge gl-absolute gl-right-0 gl-top-2': isPinnable }" > {{ pillData }} diff --git a/app/assets/javascripts/super_sidebar/components/user_bar.vue b/app/assets/javascripts/super_sidebar/components/user_bar.vue index b76ef91b768..61461164252 100644 --- a/app/assets/javascripts/super_sidebar/components/user_bar.vue +++ b/app/assets/javascripts/super_sidebar/components/user_bar.vue @@ -155,7 +155,7 @@ export default { </div> <div v-if="sidebarData.is_logged_in" - class="gl-display-flex gl-justify-content-space-between gl-px-3 gl-py-2 gl-gap-2" + class="gl-display-flex gl-justify-content-space-between gl-px-3 gl-pt-2 gl-pb-3 gl-gap-2" > <counter v-gl-tooltip:super-sidebar.hover.noninteractive.ds500.bottom="$options.i18n.issues" diff --git a/app/graphql/resolvers/codequality_reports_comparer_resolver.rb b/app/graphql/resolvers/codequality_reports_comparer_resolver.rb new file mode 100644 index 00000000000..1c034887c0d --- /dev/null +++ b/app/graphql/resolvers/codequality_reports_comparer_resolver.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Resolvers + class CodequalityReportsComparerResolver < BaseResolver + include Gitlab::Graphql::Authorize::AuthorizeResource + + type ::Types::Security::CodequalityReportsComparerType, null: true + + authorize :read_build + + def resolve + return unless Feature.enabled?(:sast_reports_in_inline_diff, object.project) + + authorize!(object.actual_head_pipeline) + + object.compare_codequality_reports + end + end +end diff --git a/app/graphql/types/merge_request_type.rb b/app/graphql/types/merge_request_type.rb index 3fe8a05b311..323d1ff79c0 100644 --- a/app/graphql/types/merge_request_type.rb +++ b/app/graphql/types/merge_request_type.rb @@ -231,6 +231,14 @@ module Types field :prepared_at, Types::TimeType, null: true, description: 'Timestamp of when the merge request was prepared.' + field :codequality_reports_comparer, + type: ::Types::Security::CodequalityReportsComparerType, + null: true, + alpha: { milestone: '16.4' }, + description: 'Code quality reports comparison reported on the merge request. Returns `null` ' \ + 'if `sast_reports_in_inline_diff` feature flag is disabled.', + resolver: ::Resolvers::CodequalityReportsComparerResolver + markdown_field :title_html, null: true markdown_field :description_html, null: true diff --git a/app/graphql/types/security/codequality_reports_comparer/degradation_type.rb b/app/graphql/types/security/codequality_reports_comparer/degradation_type.rb new file mode 100644 index 00000000000..fb7d722069f --- /dev/null +++ b/app/graphql/types/security/codequality_reports_comparer/degradation_type.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Types + module Security + module CodequalityReportsComparer + # rubocop: disable Graphql/AuthorizeTypes (The resolver authorizes the request) + class DegradationType < BaseObject + graphql_name 'CodequalityReportsComparerReportDegradation' + + description 'Represents a degradation on the compared codequality report.' + + field :description, GraphQL::Types::String, + null: false, + description: 'Description of the code quality degradation.' + + field :fingerprint, GraphQL::Types::String, + null: false, + description: 'Unique fingerprint to identify the code quality degradation. For example, an MD5 hash.' + + field :severity, Types::Ci::CodeQualityDegradationSeverityEnum, + null: false, + description: + "Severity of the code quality degradation " \ + "(#{::Gitlab::Ci::Reports::CodequalityReports::SEVERITY_PRIORITIES.keys.map(&:upcase).join(', ')})." + + field :file_path, GraphQL::Types::String, + null: false, + description: 'Relative path to the file containing the code quality degradation.' + + field :line, GraphQL::Types::Int, + null: false, + description: 'Line on which the code quality degradation occurred.' + + field :web_url, GraphQL::Types::String, + null: true, + description: 'URL to the file along with line number.' + + field :engine_name, GraphQL::Types::String, + null: false, + description: 'Code quality plugin that reported the degradation.' + end + # rubocop: enable Graphql/AuthorizeTypes + end + end +end diff --git a/app/graphql/types/security/codequality_reports_comparer/report_type.rb b/app/graphql/types/security/codequality_reports_comparer/report_type.rb new file mode 100644 index 00000000000..8a41160141a --- /dev/null +++ b/app/graphql/types/security/codequality_reports_comparer/report_type.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Types + module Security + module CodequalityReportsComparer + # rubocop: disable Graphql/AuthorizeTypes (Parent node applies authorization) + class ReportType < BaseObject + graphql_name 'CodequalityReportsComparerReport' + + description 'Represents compared code quality report.' + + field :status, + type: CodequalityReportsComparer::StatusEnum, + null: false, + description: 'Status of report.' + + field :new_errors, + type: [CodequalityReportsComparer::DegradationType], + null: false, + description: 'New code quality degradations.' + + field :resolved_errors, + type: [CodequalityReportsComparer::DegradationType], + null: true, + description: 'Resolved code quality degradations.' + + field :existing_errors, + type: [CodequalityReportsComparer::DegradationType], + null: true, + description: 'All code quality degradations.' + + field :summary, + type: CodequalityReportsComparer::SummaryType, + null: false, + description: 'Codequality report summary.' + end + # rubocop: enable Graphql/AuthorizeTypes + end + end +end diff --git a/app/graphql/types/security/codequality_reports_comparer/status_enum.rb b/app/graphql/types/security/codequality_reports_comparer/status_enum.rb new file mode 100644 index 00000000000..9cab2664db8 --- /dev/null +++ b/app/graphql/types/security/codequality_reports_comparer/status_enum.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Types + module Security + module CodequalityReportsComparer + class StatusEnum < BaseEnum + graphql_name 'CodequalityReportsComparerReportStatus' + description 'Report comparison status' + + value 'SUCCESS', value: 'success', description: 'Report successfully generated.' + value 'FAILED', value: 'failed', description: 'Report failed to generate.' + value 'NOT_FOUND', value: 'not_found', description: 'Head report or base report not found.' + end + end + end +end diff --git a/app/graphql/types/security/codequality_reports_comparer/summary_type.rb b/app/graphql/types/security/codequality_reports_comparer/summary_type.rb new file mode 100644 index 00000000000..cd4a594c193 --- /dev/null +++ b/app/graphql/types/security/codequality_reports_comparer/summary_type.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Types + module Security + module CodequalityReportsComparer + # rubocop: disable Graphql/AuthorizeTypes (The resolver authorizes the request) + class SummaryType < BaseObject + graphql_name 'CodequalityReportsComparerReportSummary' + + description 'Represents a summary of the compared codequality report.' + + field :total, + type: GraphQL::Types::Int, + null: true, + description: 'Total count of code quality degradations.' + + field :resolved, + type: GraphQL::Types::Int, + null: true, + description: 'Count of resolved code quality degradations.' + + field :errored, + type: GraphQL::Types::Int, + null: true, + description: 'Count of code quality errors.' + end + # rubocop: enable Graphql/AuthorizeTypes + end + end +end diff --git a/app/graphql/types/security/codequality_reports_comparer_type.rb b/app/graphql/types/security/codequality_reports_comparer_type.rb new file mode 100644 index 00000000000..3b0f790af81 --- /dev/null +++ b/app/graphql/types/security/codequality_reports_comparer_type.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Types + module Security + # rubocop: disable Graphql/AuthorizeTypes (The resolver authorizes the request) + class CodequalityReportsComparerType < BaseObject + graphql_name 'CodequalityReportsComparer' + + description 'Represents reports comparison for code quality.' + + field :report, + type: CodequalityReportsComparer::ReportType, + null: true, + hash_key: 'data', + description: 'Compared codequality report.' + end + # rubocop: enable Graphql/AuthorizeTypes + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 27e724f55ba..7b3fa6f8b5a 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -20,6 +20,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord dashboard_enforcement_limit dashboard_limit_new_namespace_creation_enforcement_date ], remove_with: '16.5', remove_after: '2023-08-22' + ignore_column :relay_state_domain_allowlist, remove_with: '16.6', remove_after: '2023-10-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ diff --git a/app/models/issue.rb b/app/models/issue.rb index f086430d102..0a4c95d4bab 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -14,7 +14,6 @@ class Issue < ApplicationRecord include TimeTrackable include ThrottledTouch include LabelEventable - include IgnorableColumns include MilestoneEventable include WhereComposite include StateEventable @@ -56,8 +55,6 @@ class Issue < ApplicationRecord # This default came from the enum `issue_type` column. Defined as default in the DB DEFAULT_ISSUE_TYPE = :issue - ignore_column :issue_type, remove_with: '16.4', remove_after: '2023-08-22' - belongs_to :project belongs_to :namespace, inverse_of: :issues diff --git a/app/serializers/codequality_degradation_entity.rb b/app/serializers/codequality_degradation_entity.rb index 15a26739c51..9f90a30bd2d 100644 --- a/app/serializers/codequality_degradation_entity.rb +++ b/app/serializers/codequality_degradation_entity.rb @@ -2,6 +2,9 @@ class CodequalityDegradationEntity < Grape::Entity expose :description + expose :fingerprint, if: ->(_, options) do + Feature.enabled?(:sast_reports_in_inline_diff, options[:request]&.project) + end expose :severity do |degradation| degradation.dig(:severity)&.downcase end diff --git a/app/services/system_notes/issuables_service.rb b/app/services/system_notes/issuables_service.rb index 61a4316e8ae..9d6b43634ec 100644 --- a/app/services/system_notes/issuables_service.rb +++ b/app/services/system_notes/issuables_service.rb @@ -32,8 +32,7 @@ module SystemNotes # # Returns the created Note object def relate_issuable(noteable_ref) - issuable_type = noteable.to_ability_name.humanize(capitalize: false) - body = "marked this #{issuable_type} as related to #{noteable_ref.to_reference(noteable.resource_parent)}" + body = "marked this #{noteable_name} as related to #{noteable_ref.to_reference(noteable.resource_parent)}" track_issue_event(:track_issue_related_action) @@ -534,6 +533,12 @@ module SystemNotes issue_activity_counter.public_send(event_name, author: author, project: project || noteable.project) # rubocop: disable GitlabSecurity/PublicSend end + + def noteable_name + name = noteable.try(:issue_type) || noteable.to_ability_name + + name.humanize(capitalize: false) + end end end diff --git a/config/feature_flags/development/unbatch_graphql_queries.yml b/config/feature_flags/development/unbatch_graphql_queries.yml index c5bc8c3033a..8a78a46c109 100644 --- a/config/feature_flags/development/unbatch_graphql_queries.yml +++ b/config/feature_flags/development/unbatch_graphql_queries.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/406765 milestone: '16.0' type: development group: group::project management -default_enabled: false +default_enabled: true diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index c4d59e46859..91cda3c8323 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -597,6 +597,8 @@ - 1 - - security_refresh_project_policies - 1 +- - security_scan_execution_policies_rule_schedule + - 1 - - security_scan_result_policies_sync_opened_merge_requests - 1 - - security_scan_result_policies_sync_project diff --git a/db/docs/batched_background_migrations/backfill_users_with_defaults.yml b/db/docs/batched_background_migrations/backfill_users_with_defaults.yml new file mode 100644 index 00000000000..ecc86fd665c --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_users_with_defaults.yml @@ -0,0 +1,6 @@ +--- +migration_job_name: BackfillUsersWithDefaults +description: Backfills the users table columns with their default values +feature_category: user_profile +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125881 +milestone: 16.4 diff --git a/db/post_migrate/20230816210503_ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed.rb b/db/post_migrate/20230816210503_ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed.rb new file mode 100644 index 00000000000..c3971e93526 --- /dev/null +++ b/db/post_migrate/20230816210503_ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class EnsureMrUserMentionsNoteIdBigintBackfillIsFinishedForSelfManaged < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + restrict_gitlab_migration gitlab_schema: :gitlab_main + disable_ddl_transaction! + + def up + return if com_or_dev_or_test_but_not_jh? + + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: 'merge_request_user_mentions', + column_name: 'id', + job_arguments: [['note_id'], ['note_id_convert_to_bigint']] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20230816213228_swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed.rb b/db/post_migrate/20230816213228_swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed.rb new file mode 100644 index 00000000000..93765da66dd --- /dev/null +++ b/db/post_migrate/20230816213228_swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +class SwapMergeRequestUserMentionsNoteIdToBigintForSelfManaged < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + disable_ddl_transaction! + + TABLE_NAME = 'merge_request_user_mentions' + + def up + return if com_or_dev_or_test_but_not_jh? + return if temp_column_removed?(TABLE_NAME, :note_id) + return if columns_swapped?(TABLE_NAME, :note_id) + + swap + end + + def down + return if com_or_dev_or_test_but_not_jh? + return if temp_column_removed?(TABLE_NAME, :note_id) + return unless columns_swapped?(TABLE_NAME, :note_id) + + swap + end + + def swap + # This will replace the existing index_merge_request_user_mentions_on_note_id + add_concurrent_index TABLE_NAME, :note_id_convert_to_bigint, unique: true, + name: 'index_merge_request_user_mentions_note_id_convert_to_bigint', + where: 'note_id_convert_to_bigint IS NOT NULL' + + # This will replace the existing merge_request_user_mentions_on_mr_id_and_note_id_index + add_concurrent_index TABLE_NAME, [:merge_request_id, :note_id_convert_to_bigint], unique: true, + name: 'mr_user_mentions_on_mr_id_and_note_id_convert_to_bigint_index' + + # This will replace the existing merge_request_user_mentions_on_mr_id_index + add_concurrent_index TABLE_NAME, :merge_request_id, unique: true, + name: 'merge_request_user_mentions_on_mr_id_index_convert_to_bigint', + where: 'note_id_convert_to_bigint IS NULL' + + # This will replace the existing fk_rails_c440b9ea31 + add_concurrent_foreign_key TABLE_NAME, :notes, column: :note_id_convert_to_bigint, + name: 'fk_merge_request_user_mentions_note_id_convert_to_bigint', + on_delete: :cascade + + with_lock_retries(raise_on_exhaustion: true) do + execute "LOCK TABLE notes, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE" + + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id TO note_id_tmp" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_convert_to_bigint TO note_id" + execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN note_id_tmp TO note_id_convert_to_bigint" + + function_name = Gitlab::Database::UnidirectionalCopyTrigger + .on_table(TABLE_NAME, connection: connection) + .name(:note_id, :note_id_convert_to_bigint) + execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL" + + execute 'DROP INDEX IF EXISTS index_merge_request_user_mentions_on_note_id' + rename_index TABLE_NAME, 'index_merge_request_user_mentions_note_id_convert_to_bigint', + 'index_merge_request_user_mentions_on_note_id' + + execute 'DROP INDEX IF EXISTS merge_request_user_mentions_on_mr_id_and_note_id_index' + rename_index TABLE_NAME, 'mr_user_mentions_on_mr_id_and_note_id_convert_to_bigint_index', + 'merge_request_user_mentions_on_mr_id_and_note_id_index' + + execute 'DROP INDEX IF EXISTS merge_request_user_mentions_on_mr_id_index' + rename_index TABLE_NAME, 'merge_request_user_mentions_on_mr_id_index_convert_to_bigint', + 'merge_request_user_mentions_on_mr_id_index' + + execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT IF EXISTS fk_rails_c440b9ea31" + rename_constraint(TABLE_NAME, 'fk_merge_request_user_mentions_note_id_convert_to_bigint', 'fk_rails_c440b9ea31') + end + end +end diff --git a/db/post_migrate/20230818083610_queue_backfill_users_with_defaults.rb b/db/post_migrate/20230818083610_queue_backfill_users_with_defaults.rb new file mode 100644 index 00000000000..b027ef10ddc --- /dev/null +++ b/db/post_migrate/20230818083610_queue_backfill_users_with_defaults.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueBackfillUsersWithDefaults < Gitlab::Database::Migration[2.1] + MIGRATION = "BackfillUsersWithDefaults" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 3_000 + SUB_BATCH_SIZE = 200 + MAX_BATCH_SIZE = 10_000 + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :users, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :users, :id, []) + end +end diff --git a/db/schema_migrations/20230816210503 b/db/schema_migrations/20230816210503 new file mode 100644 index 00000000000..de91e24153a --- /dev/null +++ b/db/schema_migrations/20230816210503 @@ -0,0 +1 @@ +2cdfb4fa3c4798123c14995f5dfdb1b55e163d290a566e6c6de922e22c84316a
\ No newline at end of file diff --git a/db/schema_migrations/20230816213228 b/db/schema_migrations/20230816213228 new file mode 100644 index 00000000000..9b41353e15d --- /dev/null +++ b/db/schema_migrations/20230816213228 @@ -0,0 +1 @@ +c0726a34eb21a269b85290b4aa56d05abb83d293ba7b8c9a97ea62ccba6e31c6
\ No newline at end of file diff --git a/db/schema_migrations/20230818083610 b/db/schema_migrations/20230818083610 new file mode 100644 index 00000000000..fb93a7be2a2 --- /dev/null +++ b/db/schema_migrations/20230818083610 @@ -0,0 +1 @@ +f780f4f4229f73adf3c71436b793a5b5b5c03d8cb16941e2e1f2bc5e68b67f65
\ No newline at end of file diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 5e499e302db..7a1aeefe69d 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -73,8 +73,6 @@ Inside the block you can add any of the diagrams PlantUML supports, such as: You can add parameters to block definitions: -- `format`: Can be either `png` (default) or `svg`. Use `svg` with care, as it's - not supported by all browsers, and isn't supported by Markdown. - `id`: A CSS ID added to the diagram HTML tag. - `width`: Width attribute added to the image tag. - `height`: Height attribute added to the image tag. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 6485cd6e4e0..60a07bcd32f 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -14197,6 +14197,58 @@ Code Quality report for a pipeline. | <a id="codequalityreportsummaryminor"></a>`minor` | [`Int`](#int) | Total number of minor status. | | <a id="codequalityreportsummaryunknown"></a>`unknown` | [`Int`](#int) | Total number of unknown status. | +### `CodequalityReportsComparer` + +Represents reports comparison for code quality. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="codequalityreportscomparerreport"></a>`report` | [`CodequalityReportsComparerReport`](#codequalityreportscomparerreport) | Compared codequality report. | + +### `CodequalityReportsComparerReport` + +Represents compared code quality report. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="codequalityreportscomparerreportexistingerrors"></a>`existingErrors` | [`[CodequalityReportsComparerReportDegradation!]`](#codequalityreportscomparerreportdegradation) | All code quality degradations. | +| <a id="codequalityreportscomparerreportnewerrors"></a>`newErrors` | [`[CodequalityReportsComparerReportDegradation!]!`](#codequalityreportscomparerreportdegradation) | New code quality degradations. | +| <a id="codequalityreportscomparerreportresolvederrors"></a>`resolvedErrors` | [`[CodequalityReportsComparerReportDegradation!]`](#codequalityreportscomparerreportdegradation) | Resolved code quality degradations. | +| <a id="codequalityreportscomparerreportstatus"></a>`status` | [`CodequalityReportsComparerReportStatus!`](#codequalityreportscomparerreportstatus) | Status of report. | +| <a id="codequalityreportscomparerreportsummary"></a>`summary` | [`CodequalityReportsComparerReportSummary!`](#codequalityreportscomparerreportsummary) | Codequality report summary. | + +### `CodequalityReportsComparerReportDegradation` + +Represents a degradation on the compared codequality report. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="codequalityreportscomparerreportdegradationdescription"></a>`description` | [`String!`](#string) | Description of the code quality degradation. | +| <a id="codequalityreportscomparerreportdegradationenginename"></a>`engineName` | [`String!`](#string) | Code quality plugin that reported the degradation. | +| <a id="codequalityreportscomparerreportdegradationfilepath"></a>`filePath` | [`String!`](#string) | Relative path to the file containing the code quality degradation. | +| <a id="codequalityreportscomparerreportdegradationfingerprint"></a>`fingerprint` | [`String!`](#string) | Unique fingerprint to identify the code quality degradation. For example, an MD5 hash. | +| <a id="codequalityreportscomparerreportdegradationline"></a>`line` | [`Int!`](#int) | Line on which the code quality degradation occurred. | +| <a id="codequalityreportscomparerreportdegradationseverity"></a>`severity` | [`CodeQualityDegradationSeverity!`](#codequalitydegradationseverity) | Severity of the code quality degradation (BLOCKER, CRITICAL, MAJOR, MINOR, INFO, UNKNOWN). | +| <a id="codequalityreportscomparerreportdegradationweburl"></a>`webUrl` | [`String`](#string) | URL to the file along with line number. | + +### `CodequalityReportsComparerReportSummary` + +Represents a summary of the compared codequality report. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="codequalityreportscomparerreportsummaryerrored"></a>`errored` | [`Int`](#int) | Count of code quality errors. | +| <a id="codequalityreportscomparerreportsummaryresolved"></a>`resolved` | [`Int`](#int) | Count of resolved code quality degradations. | +| <a id="codequalityreportscomparerreportsummarytotal"></a>`total` | [`Int`](#int) | Total count of code quality degradations. | + ### `Commit` #### Fields @@ -18539,6 +18591,7 @@ Defines which user roles, users, or groups can merge into a protected branch. | <a id="mergerequestautomergestrategy"></a>`autoMergeStrategy` | [`String`](#string) | Selected auto merge strategy. | | <a id="mergerequestavailableautomergestrategies"></a>`availableAutoMergeStrategies` | [`[String!]`](#string) | Array of available auto merge strategies. | | <a id="mergerequestawardemoji"></a>`awardEmoji` | [`AwardEmojiConnection`](#awardemojiconnection) | List of emoji reactions associated with the merge request. (see [Connections](#connections)) | +| <a id="mergerequestcodequalityreportscomparer"></a>`codequalityReportsComparer` **{warning-solid}** | [`CodequalityReportsComparer`](#codequalityreportscomparer) | **Introduced** in 16.4. This feature is an Experiment. It can be changed or removed at any time. Code quality reports comparison reported on the merge request. Returns `null` if `sast_reports_in_inline_diff` feature flag is disabled. | | <a id="mergerequestcommenters"></a>`commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | <a id="mergerequestcommitcount"></a>`commitCount` | [`Int`](#int) | Number of commits in the merge request. | | <a id="mergerequestcommits"></a>`commits` | [`CommitConnection`](#commitconnection) | Merge request commits. (see [Connections](#connections)) | @@ -26160,6 +26213,16 @@ Values for sorting variables. | <a id="codequalitydegradationseverityminor"></a>`MINOR` | Code Quality degradation has a status of minor. | | <a id="codequalitydegradationseverityunknown"></a>`UNKNOWN` | Code Quality degradation has a status of unknown. | +### `CodequalityReportsComparerReportStatus` + +Report comparison status. + +| Value | Description | +| ----- | ----------- | +| <a id="codequalityreportscomparerreportstatusfailed"></a>`FAILED` | Report failed to generate. | +| <a id="codequalityreportscomparerreportstatusnot_found"></a>`NOT_FOUND` | Head report or base report not found. | +| <a id="codequalityreportscomparerreportstatussuccess"></a>`SUCCESS` | Report successfully generated. | + ### `CommitActionMode` Mode of a commit action. diff --git a/doc/architecture/blueprints/cells/impact.md b/doc/architecture/blueprints/cells/impact.md index 30c70dca0cc..1f77b9056be 100644 --- a/doc/architecture/blueprints/cells/impact.md +++ b/doc/architecture/blueprints/cells/impact.md @@ -1,58 +1,7 @@ --- -stage: enablement -group: Tenant Scale -description: 'Cells: Cross-section impact' +redirect_to: 'index.md' +remove_date: '2023-11-22' --- -# Cells: Cross-section impact - -Cells is a fundamental architecture change that impacts other sections and stages. This section summarizes and links to other groups that may be impacted and highlights potential conflicts that need to be resolved. The Tenant Scale group is not responsible for achieving the goals of other groups but we want to ensure that dependencies are resolved. - -## Summary - -Based on discussions with other groups the net impact of introducing Cells and a new entity called organizations is mostly neutral. It may slow down development in some areas. We did not discover major blockers for other teams. - -1. We need to resolve naming conflicts (proposal is TBD) -1. Cells requires introducing Organizations. Organizations are a new entity **above** top-level groups. Because this is a new entity, it may impact the ability to consolidate settings for Group::Organization and influence their decision on [how to approach introducing a an organization](https://gitlab.com/gitlab-org/gitlab/-/issues/376285#approach-2-organization-is-built-on-top-of-top-level-groups) -1. Organizations may make it slightly easier for Fulfillment to realize their billing plans. - -## Impact on Group::Organization - -We synced with the Organization PM and Designer ([recording](https://youtu.be/b5Opn9cFWFk)) and discussed the similarities and differences between the Cells and Organization proposal ([presentation](https://docs.google.com/presentation/d/1FsUi22Up15b_tu6p2m-yLML3hCZ3rgrZrmzJAxUsNmU/edit?usp=sharing)). - -### Goals of Group::Organization - -As defined in the [organization documentation](../../../user/organization/index.md): - -1. Create an entity to manage everything you do as a GitLab administrator, including: - 1. Defining and applying settings to all of your groups, subgroups, and projects. - 1. Aggregating data from all your groups, subgroups, and projects. -1. Reach feature parity between SaaS and self-managed installations, with all Admin Area settings moving to groups (?). Hardware controls remain on the instance level. - -The [organization roadmap outlines](https://gitlab.com/gitlab-org/gitlab/-/issues/368237#high-level-goals) the current goals in detail. - -### Potential conflicts with Cells - -- Organization defines a new entity as the primary organizational object for groups and projects. -- We will only introduce one entity -- Group::Organization highlighted the need to further validate the key assumption that users only care about what happens within their organization. - -## Impact on Fulfillment - -We synced with Fulfillment ([recording](https://youtu.be/FkQF3uF7vTY)) to discuss how Cells would impact them. Fulfillment is supportive of an entity above top-level groups. Their perspective is outline in [!5639](https://gitlab.com/gitlab-org/customers-gitlab-com/-/merge_requests/5639/diffs). - -### Goals of Fulfillment - -- Fulfillment has a longstanding plan to move billing from the top-level group to a level above. This would mean that a license applies for an organization and all its top-level groups. -- Fulfillment uses Zuora for billing and would like to have a 1-to-1 relationship between an organization and their Zuora entity called BillingAccount. They want to move away from tying a license to a single user. -- If a customer needs multiple organizations, the corresponding BillingAccounts can be rolled up into a consolidated billing account (similar to [AWS consolidated billing](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/consolidated-billing.html)) -- Ideally, a self-managed instance has a single Organization by default, which should be enough for most customers. -- Fulfillment prefers only one additional entity. - -A rough representation of this is: - -![Cells and Fulfillment](diagrams/cells-and-fulfillment.drawio.png) - -### Potential conflicts with Cells - -- There are no known conflicts between Fulfillment's plans and Cells +This document was removed due to being outdated. +Go to [index page](index.md) for the most recent content. diff --git a/doc/architecture/blueprints/cells/impacted_features/data-pipeline-ingestion.md b/doc/architecture/blueprints/cells/impacted_features/data-pipeline-ingestion.md new file mode 100644 index 00000000000..92de3df7c27 --- /dev/null +++ b/doc/architecture/blueprints/cells/impacted_features/data-pipeline-ingestion.md @@ -0,0 +1,39 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Data Pipeline Ingestion' +--- + +<!-- vale gitlab.FutureTense = NO --> + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Data pipeline ingestion + +The Cells architecture will have a significant impact on the current [data pipeline](https://about.gitlab.com/handbook/business-technology/data-team/platform/pipelines/SAAS-Gitlab-com/) which exports data from Postgres to Snowflake for the use of data analytics. This data pipeline fulfils many use cases (i.e. SAAS Service ping, Gainsight metrics and Reporting and Analytics of the SAAS Platform). + +## 1. Definition + +## 2. Data flow + +The current data pipeline is limited by not having the possibility to get data via a CDC mechanism (which leads to data quality issues) and works by polling the Postgres database and looking for new and updated records or fully extracting data for certain tables which causes a lot of overhead. +At the moment the data pipeline runs against two instances that get created from a snapshot of both the `main` and `ci` databases. +This is done to avoid workload on the production databases. +In the Cells architecture there will be more Postgres instances because of which the current pipeline couldn't scale to pull data from all the Postgres instances. Requirements around the data pipeline moving forward are as follows: + +- We need a process that allows capturing all the CDC (insert, update and delete) from all Cells, scaling automatically with N number of Cells. +- We need to have (direct or indirect) access to database instances which allows it to do data catch up in case of major failure or root cause analysis for data anomalies. +- We need monitoring in place to alert any incident that can delay the data ingestion. + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/index.md b/doc/architecture/blueprints/cells/index.md index 5ec34a5036d..2f662dc65a6 100644 --- a/doc/architecture/blueprints/cells/index.md +++ b/doc/architecture/blueprints/cells/index.md @@ -20,7 +20,6 @@ For more information about Cells, see also: - [Glossary](glossary.md) - [Goals](goals.md) -- [Cross-section impact](impact.md) ## Work streams @@ -310,19 +309,6 @@ This section links all different technical proposals that are being evaluated. - [Stateless Router That Uses a Cache to Pick Cell and Is Redirected When Wrong Cell Is Reached](proposal-stateless-router-with-buffering-requests.md) - [Stateless Router That Uses a Cache to Pick Cell and pre-flight `/api/v4/cells/learn`](proposal-stateless-router-with-routes-learning.md) -## Data pipeline ingestion - -The Cells architecture will have a significant impact on the current [data pipeline](https://about.gitlab.com/handbook/business-technology/data-team/platform/pipelines/SAAS-Gitlab-com/) which exports data from Postgres to Snowflake for the use of data analytics. This data pipeline fulfils many use cases (i.e. SAAS Service ping, Gainsight metrics and Reporting and Analytics of the SAAS Platform). - -The current data pipeline is limited by not having the possibility to get data via a CDC mechanism (which leads to data quality issues) and works by polling the Postgres database and looking for new and updated records or fully extracting data for certain tables which causes a lot of overhead. -At the moment the data pipeline runs against two instances that get created from a snapshot of both the `main` and `ci` databases. -This is done to avoid workload on the production databases. -In the Cells architecture there will be more Postgres instances because of which the current pipeline couldn't scale to pull data from all the Postgres instances. Requirements around the data pipeline moving forward are as follows: - -- We need a process that allows capturing all the CDC (insert, update and delete) from all Cells, scaling automatically with N number of Cells. -- We need to have (direct or indirect) access to database instances which allows it to do data catch up in case of major failure or root cause analysis for data anomalies. -- We need monitoring in place to alert any incident that can delay the data ingestion. - ## Impacted features The Cells architecture will impact many features requiring some of them to be rewritten, or changed significantly. @@ -356,6 +342,7 @@ The following list of impacted features only represents placeholders that still - [Cells: Router Endpoints Classification](impacted_features/router-endpoints-classification.md) - [Cells: Schema changes (Postgres and Elasticsearch migrations)](impacted_features/schema-changes.md) - [Cells: Uploads](impacted_features/uploads.md) +- [Cells: Data pipeline ingestion](impacted_features/data-pipeline-ingestion.md) - ... ## Frequently Asked Questions diff --git a/doc/ci/pipelines/pipeline_architectures.md b/doc/ci/pipelines/pipeline_architectures.md index d0324f16ffb..305e48ca9f5 100644 --- a/doc/ci/pipelines/pipeline_architectures.md +++ b/doc/ci/pipelines/pipeline_architectures.md @@ -306,5 +306,4 @@ deploy_b: environment: production ``` -It's also possible to set jobs to run before or after triggering child pipelines, -for example if you have common setup steps or a unified deployment at the end. +Jobs can be set to run before or after triggering child pipelines in GitLab, allowing common setup steps or unified deployment. diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index 10490df7b5e..a6d827df820 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -193,9 +193,10 @@ Because batched migrations are update heavy and there were few incidents in the These database indicators are checked to throttle a migration. On getting a stop signal, the migration is paused for a set time (10 minutes): -- WAL queue pending archival crossing a threshold. +- WAL queue pending archival crossing the threshold. - Active autovacuum on the tables on which the migration works on. - Patroni apdex SLI dropping below the SLO. +- WAL rate crossing the threshold. It's an ongoing effort to add more indicators to further enhance the database health check framework. For more details, see diff --git a/doc/development/database_review.md b/doc/development/database_review.md index bb0bfbc759b..cea0945de2d 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -173,10 +173,11 @@ Include in the MR description: ##### Query Plans - The query plan for each raw SQL query included in the merge request along with the link to the query plan following each raw SQL snippet. -- Provide a link to the plan generated using the `explain` command in the [postgres.ai](database/database_lab.md) chatbot. - - If it's not possible to get an accurate picture in Database Lab, you may need to seed a development environment, and instead provide links - from [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com). Be sure to paste both the plan - and the query used in the form. +- Provide a link to the plan generated using the `explain` command in the [postgres.ai](database/database_lab.md) chatbot. The `explain` command runs + `EXPLAIN ANALYZE`. + - If it's not possible to get an accurate picture in Database Lab, you may need to + seed a development environment, and instead provide output + from `EXPLAIN ANALYZE`. Create links to the plan using [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com). Be sure to paste both the plan and the query used in the form. - When providing query plans, make sure it hits enough data: - To produce a query plan with enough data, you can use the IDs of: - The `gitlab-org` namespace (`namespace_id = 9970`), for queries involving a group. diff --git a/doc/development/internal_analytics/internal_event_tracking/migration.md b/doc/development/internal_analytics/internal_event_tracking/migration.md new file mode 100644 index 00000000000..7566c44a1c8 --- /dev/null +++ b/doc/development/internal_analytics/internal_event_tracking/migration.md @@ -0,0 +1,155 @@ +--- +stage: Analytics +group: Analytics Instrumentation +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Migrating existing tracking to internal event tracking + +GitLab Internal Events Tracking exposes a unified API on top of the existing tracking options. Currently RedisHLL and Snowplow are supported. + +This page describes how you can switch from tracking using a single method to using Internal Events Tracking. + +## Migrating from tracking with Snowplow + +If you are already tracking events in Snowplow, you can start collecting metrics also from self-managed instances by switching to Internal Events Tracking. + +Notice that the Snowplow event you trigger after switching to Internal Events Tracking looks slightly different from your current event. + +Please make sure that you are okay with this change before you migrate. + +### Backend + +If you are already tracking Snowplow events using `Gitlab::Tracking.event` and you want to migrate to Internal Events Tracking you might start with something like this: + +```ruby +Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, + project: project, context: [context], user: user, label: label) +``` + +The code above can be replaced by something like this: + +```ruby +Gitlab::InternalEvents.track_event('ci_templates_unique', namespace: namespace, project: project, user: user) +``` + +In addition, you have to create definitions for the metrics that you would like to track. + +To generate metric definitions, you can use the generator like this: + +```shell +bin/rails g gitlab:analytics:internal_events \ + --time_frames=7d 28d\ + --group=project_management \ + --stage=plan \ + --section=dev \ + --event=ci_templates_unique \ + --unique=user.id \ + --mr=https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121544 +``` + +### Frontend + +If you are using the `Tracking` mixin in the Vue component, you can replace it with the `InternalEvents` mixin. + +For example, if your current Vue component look like this: + +```vue +import Tracking from '~/tracking'; +... +mixins: [Tracking.mixin()] +... +... +this.track('some_label', options) +``` + +After converting it to Internal Events Tracking, it should look like this: + +```vue +import { InternalEvents } from '~/tracking'; +... +mixins: [InternalEvents.mixin()] +... +... +this.track_event('action') +``` + +You can use [this MR](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123901/diffs) as an example. It migrates the `devops_adoption_app` component to use Internal Events Tracking. + +If you are using `data-track-action` in the component, you have to change it to `data-event-tracking` to migrate to Internal Events Tracking. + +For example, if a button is defined like this: + +```vue + <gl-button + :href="diffFile.external_url" + :title="externalUrlLabel" + :aria-label="externalUrlLabel" + target="_blank" + data-track-action="click_toggle_external_button" + data-track-label="diff_toggle_external_button" + data-track-property="diff_toggle_external" + icon="external-link" +/> +``` + +This can be converted to Internal Events Tracking like this: + +```vue + <gl-button + :href="diffFile.external_url" + :title="externalUrlLabel" + :aria-label="externalUrlLabel" + target="_blank" + data-event-tracking="click_toggle_external_button" + icon="external-link" +/> +``` + +Notice that we just need action to pass in the `data-event-tracking` attribute which will be passed to both Snowplow and RedisHLL. + +## Migrating from tracking with RedisHLL + +### Backend + +If you are currently tracking a metric in `RedisHLL` like this: + +```ruby + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:git_write_action, values: current_user.id) +``` + +To start using Internal Events Tracking, follow these steps: + +1. Create an event definition that describes `git_write_action` ([guide](../snowplow/event_dictionary_guide.md#create-a-new-event-definition)). +1. Find metric definitions that list `git_write_action` in the events section (`20210216182041_action_monthly_active_users_git_write.yml` and `20210216184045_git_write_action_weekly.yml`). +1. Change the `data_source` from `redis_hll` to `internal_events` in the metric definition files. +1. Add an `events` section to both metric definition files. + + ```yaml + events: + - name: git_write_action + unique: user.id + ``` + + Use `project.id` or `namespace.id` instead of `user.id` if your metric is counting something other than unique users. +1. Call `InternalEvents.tract_event` instead of `HLLRedisCounter.track_event`: + + ```diff + - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:git_write_action, values: current_user.id) + + Gitlab::InternalEvents.track_event('project_created', user: current_user) + ``` + +1. Optional. Add additional values to the event. You typically want to add `project` and `namespace` as it is useful information to have in the data warehouse. + + ```diff + - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:git_write_action, values: current_user.id) + + Gitlab::InternalEvents.track_event('project_created', user: current_user, project: project, namespace: namespace) + ``` + +1. Update your test to use the `internal event tracking` shared example. + +### Frontend + +If you are calling `trackRedisHllUserEvent` in the frontend to track the frontend event, you can convert this to Internal events by using mixin, raw JavaScript or data tracking attribute, + +[Quick start guide](quick_start.md#frontend-tracking) has example for each methods. diff --git a/doc/development/internal_analytics/internal_event_tracking/quick_start.md b/doc/development/internal_analytics/internal_event_tracking/quick_start.md index 84926657a3b..bdd6c1f8f6e 100644 --- a/doc/development/internal_analytics/internal_event_tracking/quick_start.md +++ b/doc/development/internal_analytics/internal_event_tracking/quick_start.md @@ -4,12 +4,36 @@ group: Analytics Instrumentation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Quick start for internal event tracking +# Quick start for Internal Event Tracking In an effort to provide a more efficient, scalable, and unified tracking API, GitLab is deprecating existing RedisHLL and Snowplow tracking. Instead, we're implementing a new `track_event` method. -With this approach, we can both update RedisHLL counters and send Snowplow events simultaneously, streamlining the tracking process. +With this approach, we can update both RedisHLL counters and send Snowplow events without worrying about the underlying implementation. -## Create and trigger events +In order to instrument your code with Internal Events Tracking need three things: + +1. Define an event +1. Define one or more metrics +1. Trigger the event + +## Defining event and metrics + +To create event and metric definitions you can use the `internal_events` generator. + +This example will create an event definition for an event called `project_created` and two metric definitions which will be aggregated every 7 and 28 days. + +```shell +bin/rails g gitlab:analytics:internal_events \ +--time_frames=7d 28d \ +--group=project_management \ +--stage=plan --section=dev \ +--event=project_created \ +--unique=user.id \ +--mr=https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121544 +``` + +## Trigger events + +Triggering an event and thereby updating a metric is slightly different on backend and frontend. Please refer to the relevant section below. ### Backend tracking @@ -18,9 +42,9 @@ To trigger an event, call the `Gitlab::InternalEvents.track_event` method with t ```ruby Gitlab::InternalEvents.track_event( "i_code_review_user_apply_suggestion", - user_id: user_id, - namespace_id: namespace_id, - project_id: project_id + user: user, + namespace: namespace, + project: project ) ``` @@ -46,7 +70,7 @@ To implement Vue component tracking: ```javascript export default { mixins: [trackingMixin], - + data() { return { expanded: false, @@ -97,7 +121,7 @@ This attribute ensures that if we want to track GitLab internal events for a but </gl-button> ``` -For Haml +#### Haml ```haml = render Pajamas::ButtonComponent.new(button_options: { class: 'js-settings-toggle', data: { event_tracking: 'action' }}) do @@ -111,3 +135,7 @@ Sometimes we want to send internal events when the component is rendered or load = render Pajamas::ButtonComponent.new(button_options: { data: { event_tracking_load: 'true', event_tracking: 'i_devops' } }) do = _("New project") ``` + +### Limitations + +The only values we allow for `unique` are `user.id`, `project.id`, and `namespace.id`, as they are logged as part of the standard context. We currently don't have anywhere to put a value like `merge_request.id`. That will change with self-describing events. diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 813a045d55c..8804cb36de2 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -152,7 +152,7 @@ that are scoped to a single [configuration keyword](../../ci/yaml/index.md#job-k | `.setup-test-env-cache` | Allows a job to use a default `cache` definition suitable for setting up test environment for subsequent Ruby/Rails tasks. | | `.ruby-cache` | Allows a job to use a default `cache` definition suitable for Ruby tasks. | | `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. | -| `.coverage-cache` | Allows a job to use a default `cache` definition suitable for coverage tasks. | +| `.ruby-gems-coverage-cache` | Allows a job to use a default `cache` definition suitable for coverage tasks. | | `.qa-cache` | Allows a job to use a default `cache` definition suitable for QA tasks. | | `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. | | `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. | diff --git a/doc/development/pipelines/performance.md b/doc/development/pipelines/performance.md index 2dbed640fbb..7db498adc02 100644 --- a/doc/development/pipelines/performance.md +++ b/doc/development/pipelines/performance.md @@ -40,7 +40,7 @@ This works well for the following reasons: - `.ruby-cache` - `.static-analysis-cache` - `.rubocop-cache` - - `.coverage-cache` + - `.ruby-gems-coverage-cache` - `.ruby-node-cache` - `.qa-cache` - `.yarn-cache` diff --git a/doc/user/application_security/dependency_list/index.md b/doc/user/application_security/dependency_list/index.md index a55f26f529d..944c5eac709 100644 --- a/doc/user/application_security/dependency_list/index.md +++ b/doc/user/application_security/dependency_list/index.md @@ -100,6 +100,9 @@ GitLab displays dependencies with the following information: Displayed dependencies are initially sorted by packager. They can also be sorted by name. +NOTE: +The project search feature is only supported on groups that have up to 600 occurrences within their group hierarchy. + ## Downloading the dependency list You can download the full list of dependencies and their details in diff --git a/lib/gitlab/background_migration/backfill_users_with_defaults.rb b/lib/gitlab/background_migration/backfill_users_with_defaults.rb new file mode 100644 index 00000000000..10b4963163d --- /dev/null +++ b/lib/gitlab/background_migration/backfill_users_with_defaults.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the users table columns with their default values + class BackfillUsersWithDefaults < BatchedMigrationJob + operation_name :backfill_users_with_defaults + feature_category :user_profile + + def perform + each_sub_batch do |sub_batch| + connection.transaction do + sub_batch.where(project_view: nil).update_all(project_view: 2) + sub_batch.where(hide_no_ssh_key: nil).update_all(hide_no_ssh_key: false) + sub_batch.where(hide_no_password: nil).update_all(hide_no_password: false) + sub_batch.where(notified_of_own_activity: nil).update_all(notified_of_own_activity: false) + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index fe76543548b..905588c2afc 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -553,7 +553,8 @@ module Gitlab message: encode_binary(message), start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository, - dry_run: dry_run + dry_run: dry_run, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) gitaly_client_call( diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 7b6eee7182f..34a79393a0e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -15834,6 +15834,9 @@ msgstr "" msgid "Dependencies|Packager" msgstr "" +msgid "Dependencies|Project list unavailable" +msgstr "" + msgid "Dependencies|Projects" msgstr "" @@ -15852,6 +15855,9 @@ msgstr "" msgid "Dependencies|There may be multiple paths" msgstr "" +msgid "Dependencies|This group exceeds the maximum number of sub-groups of 600. We cannot accurately display a project list at this time. Please access a sub-group dependency list to view this information or see the %{linkStart}dependency list help %{linkEnd} page to learn more." +msgstr "" + msgid "Dependencies|Toggle vulnerability list" msgstr "" diff --git a/qa/Gemfile b/qa/Gemfile index 8ad40ec12c2..0dfcb09c8e0 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -2,7 +2,7 @@ source 'https://rubygems.org' -gem 'gitlab-qa', '~> 12', '>= 12.2.1', require: 'gitlab/qa' +gem 'gitlab-qa', '~> 12', '>= 12.3.0', require: 'gitlab/qa' gem 'gitlab_quality-test_tooling', '~> 0.9.3', require: false gem 'gitlab-utils', path: '../gems/gitlab-utils' gem 'activesupport', '~> 7.0.5.1' # This should stay in sync with the root's Gemfile diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 67d64684aa1..c02f82402bc 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -121,7 +121,7 @@ GEM gitlab (4.19.0) httparty (~> 0.20) terminal-table (>= 1.5.1) - gitlab-qa (12.2.1) + gitlab-qa (12.3.0) activesupport (>= 6.1, < 7.1) gitlab (~> 4.19) http (~> 5.0) @@ -351,7 +351,7 @@ DEPENDENCIES faraday-retry (~> 2.2) fog-core (= 2.1.0) fog-google (~> 1.19) - gitlab-qa (~> 12, >= 12.2.1) + gitlab-qa (~> 12, >= 12.3.0) gitlab-utils! gitlab_quality-test_tooling (~> 0.9.3) influxdb-client (~> 2.9) diff --git a/qa/perf/ai/Dockerfile.ab b/qa/perf/ai/Dockerfile.ab new file mode 100644 index 00000000000..72c4681ed47 --- /dev/null +++ b/qa/perf/ai/Dockerfile.ab @@ -0,0 +1,7 @@ +FROM alpine:3.18 + +RUN apk add --no-cache apache2-utils + +COPY prompt.json . + +ENTRYPOINT [ "/usr/bin/ab" ] diff --git a/qa/perf/ai/README.md b/qa/perf/ai/README.md new file mode 100644 index 00000000000..c0fbcf09f0c --- /dev/null +++ b/qa/perf/ai/README.md @@ -0,0 +1,43 @@ +# Performance testing tools for AI features + +## Code Suggestions + +These tests are based on [examples from testing of the Code Suggestions Model Gateway](https://gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist/-/issues/77). + +### Apache Bench + +If you're using a Mac you may already have the [Apache Bench](https://httpd.apache.org/docs/current/programs/ab.html) tool installed: + +```shell +$ which ab +/usr/sbin/ab +``` + +#### Build a Docker image + +If you don't already have Apache Bench installed, [`Dockerfile.ab`](Dockerfile.ab) allows you to build a Docker image that you can use to run it. + +To build the image, from the `qa/perf/ai` directory run: + +```shell +docker build -t ab -f Dockerfile.ab . +``` + +That will create an image named `ab`. When you start a container via `docker run <image>` any arguments that follow will be passed to `ab` inside +the container. The command in the section below assumes you have Apache Bench installed, but you can use the image you built if you +add `docker run --rm` in front of the command. For example, instead of `ab -n 1000 ...` use `docker run --rm ab -n 1000 ...`. + +### Run tests + +WARNING: +Be mindful of the potential impact before running a performance test. It could disrupt accessibility for other users and could be problematic for our use of third-party services. If you're unsure, consult the relevant teams (those working on the Code Suggestions feature, the [AI-powered:AI Framework](https://about.gitlab.com/handbook/product/categories/#ai-framework-group) group, the [Create:Code Creation](https://about.gitlab.com/handbook/product/categories/#code-creation-group) group, and the [`#g_code_suggestions` Slack channel (internal)](https://gitlab.slack.com/archives/C048Z2DHWGP)) + +To run performance tests using Apache Bench, execute the following from the `qa/perf/ai` directory: + +```shell +export GITLAB_PAT=<your personal access token> +export URL=<the code suggestions endpoint to test> +ab -n 1000 -c 20 -H "Authorization: Bearer $GITLAB_PAT" -T 'application/json' -p prompt.json $URL +``` + +For example, if `$URL` were `https://staging.gitlab.com/api/v4/code_suggestions/completions` that command would `POST` 1000 requests, 20 at a time, to the staging Code Suggestions completions endpoint, using the data in [`prompt.json`](prompt.json) as the prompt. diff --git a/qa/perf/ai/prompt.json b/qa/perf/ai/prompt.json new file mode 100644 index 00000000000..8668f71f4f5 --- /dev/null +++ b/qa/perf/ai/prompt.json @@ -0,0 +1,10 @@ +{ + "prompt_version": 1, + "project_path": "gitlab-org/gitlab", + "project_id": 278964, + "current_file": { + "file_name": "test.py", + "content_above_cursor": "import pytest", + "content_below_cursor": "" + } +} diff --git a/spec/fixtures/api/schemas/entities/codequality_degradation.json b/spec/fixtures/api/schemas/entities/codequality_degradation.json index ac772873daf..ac1a556e33c 100644 --- a/spec/fixtures/api/schemas/entities/codequality_degradation.json +++ b/spec/fixtures/api/schemas/entities/codequality_degradation.json @@ -10,6 +10,9 @@ "description": { "type": "string" }, + "fingerprint": { + "type": "string" + }, "severity": { "type": "string" }, diff --git a/spec/graphql/types/merge_request_type_spec.rb b/spec/graphql/types/merge_request_type_spec.rb index bd271da55a9..c8545d811fa 100644 --- a/spec/graphql/types/merge_request_type_spec.rb +++ b/spec/graphql/types/merge_request_type_spec.rb @@ -36,7 +36,7 @@ RSpec.describe GitlabSchema.types['MergeRequest'], feature_category: :code_revie commit_count current_user_todos conflicts auto_merge_enabled approved_by source_branch_protected squash_on_merge available_auto_merge_strategies has_ci mergeable commits committers commits_without_merge_commits squash security_auto_fix default_squash_commit_message - auto_merge_strategy merge_user award_emoji prepared_at + auto_merge_strategy merge_user award_emoji prepared_at codequality_reports_comparer ] expect(described_class).to have_graphql_fields(*expected_fields).at_least diff --git a/spec/graphql/types/security/codequality_reports_comparer/degradation_type_spec.rb b/spec/graphql/types/security/codequality_reports_comparer/degradation_type_spec.rb new file mode 100644 index 00000000000..9ae1b3fcf94 --- /dev/null +++ b/spec/graphql/types/security/codequality_reports_comparer/degradation_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CodequalityReportsComparerReportDegradation'], feature_category: :code_quality do + specify { expect(described_class.graphql_name).to eq('CodequalityReportsComparerReportDegradation') } + + it 'has expected fields' do + expected_fields = %i[description fingerprint severity file_path line web_url engine_name] + + expect(described_class).to have_graphql_fields(*expected_fields) + end +end diff --git a/spec/graphql/types/security/codequality_reports_comparer/report_type_spec.rb b/spec/graphql/types/security/codequality_reports_comparer/report_type_spec.rb new file mode 100644 index 00000000000..a1788126f1b --- /dev/null +++ b/spec/graphql/types/security/codequality_reports_comparer/report_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CodequalityReportsComparerReport'], feature_category: :code_quality do + specify { expect(described_class.graphql_name).to eq('CodequalityReportsComparerReport') } + + it 'has expected fields' do + expected_fields = %i[status new_errors resolved_errors existing_errors summary] + + expect(described_class).to have_graphql_fields(*expected_fields) + end +end diff --git a/spec/graphql/types/security/codequality_reports_comparer/status_enum_spec.rb b/spec/graphql/types/security/codequality_reports_comparer/status_enum_spec.rb new file mode 100644 index 00000000000..6e5bdd1e91d --- /dev/null +++ b/spec/graphql/types/security/codequality_reports_comparer/status_enum_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CodequalityReportsComparerReportStatus'], feature_category: :code_quality do + specify { expect(described_class.graphql_name).to eq('CodequalityReportsComparerReportStatus') } + + it 'exposes all codequality report status values' do + expect(described_class.values.keys).to contain_exactly('SUCCESS', 'FAILED', 'NOT_FOUND') + end +end diff --git a/spec/graphql/types/security/codequality_reports_comparer/summary_type_spec.rb b/spec/graphql/types/security/codequality_reports_comparer/summary_type_spec.rb new file mode 100644 index 00000000000..41de93b27ad --- /dev/null +++ b/spec/graphql/types/security/codequality_reports_comparer/summary_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CodequalityReportsComparerReportSummary'], feature_category: :code_quality do + specify { expect(described_class.graphql_name).to eq('CodequalityReportsComparerReportSummary') } + + it 'has expected fields' do + expected_fields = %i[total resolved errored] + + expect(described_class).to have_graphql_fields(*expected_fields) + end +end diff --git a/spec/graphql/types/security/codequality_reports_comparer_type_spec.rb b/spec/graphql/types/security/codequality_reports_comparer_type_spec.rb new file mode 100644 index 00000000000..02f7a9d6925 --- /dev/null +++ b/spec/graphql/types/security/codequality_reports_comparer_type_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CodequalityReportsComparer'], feature_category: :code_quality do + specify { expect(described_class.graphql_name).to eq('CodequalityReportsComparer') } + + it 'has expected fields' do + expect(described_class).to have_graphql_fields(:report) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb b/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb new file mode 100644 index 00000000000..78f36933435 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_users_with_defaults_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillUsersWithDefaults, + schema: 20230818083610, + feature_category: :user_profile do + let(:users) { table(:users) } + let(:columns) { [:project_view, :hide_no_ssh_key, :hide_no_password, :notified_of_own_activity] } + let(:initial_column_values) do + [ + [nil, nil, nil, nil], + [0, nil, nil, nil], + [nil, true, nil, nil], + [nil, nil, true, nil], + [nil, nil, nil, true] + ] + .map { |row| columns.zip(row).to_h } + end + + let(:final_column_values) do + [ + [2, false, false, false], + [0, false, false, false], + [2, true, false, false], + [2, false, true, false], + [2, false, false, true] + ] + .map { |row| columns.zip(row).to_h } + end + + subject(:perform_migration) do + described_class + .new( + start_id: users.minimum(:id), + end_id: users.maximum(:id), + batch_table: :users, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + .perform + end + + before do + initial_column_values.each_with_index do |attributes, index| + user = users.create!(**attributes.merge(projects_limit: 1, email: "user#{index}@gitlab.com")) + final_column_values[index].merge!(id: user.id) + end + end + + it 'backfills the null values with the default values' do + perform_migration + + final_column_values.each { |attributes| match_attributes(attributes) } + end + + private + + def match_attributes(attributes) + migrated_user = users.find(attributes[:id]) + expect(migrated_user.project_view).to eq(attributes[:project_view]) + expect(migrated_user.hide_no_ssh_key).to eq(attributes[:hide_no_ssh_key]) + expect(migrated_user.hide_no_password).to eq(attributes[:hide_no_password]) + expect(migrated_user.notified_of_own_activity).to eq(attributes[:notified_of_own_activity]) + end +end diff --git a/spec/migrations/20230818083610_queue_backfill_users_with_defaults_spec.rb b/spec/migrations/20230818083610_queue_backfill_users_with_defaults_spec.rb new file mode 100644 index 00000000000..4cd72b4fa7a --- /dev/null +++ b/spec/migrations/20230818083610_queue_backfill_users_with_defaults_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillUsersWithDefaults, feature_category: :user_profile do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :users, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + max_batch_size: described_class::MAX_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/migrations/ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed_spec.rb b/spec/migrations/ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed_spec.rb new file mode 100644 index 00000000000..482cca67e46 --- /dev/null +++ b/spec/migrations/ensure_mr_user_mentions_note_id_bigint_backfill_is_finished_for_self_managed_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe EnsureMrUserMentionsNoteIdBigintBackfillIsFinishedForSelfManaged, feature_category: :database do + describe '#up' do + let(:migration_arguments) do + { + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: 'merge_request_user_mentions', + column_name: 'id', + job_arguments: [['note_id'], ['note_id_convert_to_bigint']] + } + end + + it 'ensures the migration is completed for self-managed instances' do + expect_next_instance_of(described_class) do |instance| + expect(instance).to receive(:com_or_dev_or_test_but_not_jh?).and_return(false) + expect(instance).to receive(:ensure_batched_background_migration_is_finished).with(migration_arguments) + end + + migrate! + end + + it 'skips the check for GitLab.com, dev, or test' do + expect_next_instance_of(described_class) do |instance| + expect(instance).to receive(:com_or_dev_or_test_but_not_jh?).and_return(true) + expect(instance).not_to receive(:ensure_batched_background_migration_is_finished) + end + + migrate! + end + end +end diff --git a/spec/migrations/swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed_spec.rb b/spec/migrations/swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed_spec.rb new file mode 100644 index 00000000000..5cba691e20e --- /dev/null +++ b/spec/migrations/swap_merge_request_user_mentions_note_id_to_bigint_for_self_managed_spec.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe SwapMergeRequestUserMentionsNoteIdToBigintForSelfManaged, feature_category: :database do + let(:connection) { described_class.new.connection } + let(:merge_request_user_mentions) { table(:merge_request_user_mentions) } + + shared_examples 'column `note_id_convert_to_bigint` is already dropped' do + before do + connection.execute('ALTER TABLE merge_request_user_mentions ALTER COLUMN note_id TYPE bigint') + connection.execute('ALTER TABLE merge_request_user_mentions DROP COLUMN IF EXISTS note_id_convert_to_bigint') + end + + after do + connection.execute('ALTER TABLE merge_request_user_mentions DROP COLUMN IF EXISTS note_id_convert_to_bigint') + end + + it 'does not swaps the columns' do + disable_migrations_output do + reversible_migration do |migration| + migration.before -> { + merge_request_user_mentions.reset_column_information + + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('bigint') + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id_convert_to_bigint' }).to be_nil + } + + migration.after -> { + merge_request_user_mentions.reset_column_information + + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('bigint') + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id_convert_to_bigint' }).to be_nil + } + end + end + end + end + + describe '#up' do + before do + # rubocop:disable RSpec/AnyInstanceOf + allow_any_instance_of(described_class).to( + receive(:com_or_dev_or_test_but_not_jh?).and_return(com_or_dev_or_test_but_not_jh?) + ) + # rubocop:enable RSpec/AnyInstanceOf + end + + context 'when GitLab.com, dev, or test' do + let(:com_or_dev_or_test_but_not_jh?) { true } + + it_behaves_like 'column `note_id_convert_to_bigint` is already dropped' + end + + context 'when self-managed instance with the `note_id_convert_to_bigint` column already dropped' do + let(:com_or_dev_or_test_but_not_jh?) { false } + + it_behaves_like 'column `note_id_convert_to_bigint` is already dropped' + end + + context 'when self-managed instance columns already swapped' do + let(:com_or_dev_or_test_but_not_jh?) { false } + + before do + connection.execute('ALTER TABLE merge_request_user_mentions ALTER COLUMN note_id TYPE bigint') + connection.execute( + 'ALTER TABLE merge_request_user_mentions ADD COLUMN IF NOT EXISTS note_id_convert_to_bigint integer' + ) + + disable_migrations_output { migrate! } + end + + after do + connection.execute('ALTER TABLE merge_request_user_mentions DROP COLUMN IF EXISTS note_id_convert_to_bigint') + end + + it 'does not swaps the columns' do + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('bigint') + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id_convert_to_bigint' }.sql_type).to( + eq('integer') + ) + end + end + + context 'when self-managed instance' do + let(:com_or_dev_or_test_but_not_jh?) { false } + + before do + connection.execute('ALTER TABLE merge_request_user_mentions ALTER COLUMN note_id TYPE integer') + connection.execute( + 'ALTER TABLE merge_request_user_mentions ADD COLUMN IF NOT EXISTS note_id_convert_to_bigint bigint' + ) + connection.execute('ALTER TABLE merge_request_user_mentions ALTER COLUMN note_id_convert_to_bigint TYPE bigint') + connection.execute( + 'DROP INDEX IF EXISTS index_merge_request_user_mentions_on_note_id_convert_to_bigint CASCADE' + ) + connection.execute('CREATE OR REPLACE FUNCTION trigger_bfcbace4260d() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN NEW."note_id_convert_to_bigint" := NEW."note_id"; RETURN NEW; END; $$;') + end + + after do + connection.execute('ALTER TABLE merge_request_user_mentions DROP COLUMN IF EXISTS note_id_convert_to_bigint') + end + + it 'swaps the columns' do + disable_migrations_output do + reversible_migration do |migration| + migration.before -> { + merge_request_user_mentions.reset_column_information + + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('integer') + expect(merge_request_user_mentions.columns.find do |c| + c.name == 'note_id_convert_to_bigint' + end.sql_type).to( + eq('bigint') + ) + } + + migration.after -> { + merge_request_user_mentions.reset_column_information + + expect(merge_request_user_mentions.columns.find { |c| c.name == 'note_id' }.sql_type).to eq('bigint') + expect(merge_request_user_mentions.columns.find do |c| + c.name == 'note_id_convert_to_bigint' + end.sql_type).to( + eq('integer') + ) + } + end + end + end + end + end +end diff --git a/spec/requests/api/graphql/merge_requests/codequality_reports_comparer_spec.rb b/spec/requests/api/graphql/merge_requests/codequality_reports_comparer_spec.rb new file mode 100644 index 00000000000..2939e9307e9 --- /dev/null +++ b/spec/requests/api/graphql/merge_requests/codequality_reports_comparer_spec.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Query.project.mergeRequest.codequalityReportsComparer', feature_category: :code_quality do + include GraphqlHelpers + + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:merge_request) { create(:merge_request, :with_codequality_reports, source_project: project) } + + let(:mock_report) do + { + status: :parsed, + data: { + status: 'failed', + new_errors: [ + { + description: "Method `new_array` has 12 arguments (exceeds 4 allowed). Consider refactoring.", + fingerprint: "15cdb5c53afd42bc22f8ca366a08d547", + severity: "major", + file_path: "foo.rb", + line: 10, + engine_name: "structure" + }, + { + description: "Method `backwards_array` has 12 arguments (exceeds 4 allowed). Consider refactoring.", + fingerprint: "f3bdc1e8c102ba5fbd9e7f6cda51c95e", + severity: "major", + file_path: "foo.rb", + line: 14, + engine_name: "structure" + }, + { + description: "Avoid parameter lists longer than 5 parameters. [12/5]", + fingerprint: "ab5f8b935886b942d621399f5a2ca16e", + severity: "minor", + file_path: "foo.rb", + line: 14, + engine_name: "rubocop" + } + ], + resolved_errors: [], + existing_errors: [], + summary: { + total: 3, + resolved: 0, + errored: 3 + } + }.deep_stringify_keys + } + end + + let(:codequality_reports_comparer_fields) do + <<~QUERY + codequalityReportsComparer { + report { + status + newErrors { + description + fingerprint + severity + filePath + line + webUrl + engineName + } + resolvedErrors { + description + fingerprint + severity + filePath + line + webUrl + engineName + } + existingErrors { + description + fingerprint + severity + filePath + line + webUrl + engineName + } + summary { + errored + resolved + total + } + } + } + QUERY + end + + let(:merge_request_fields) do + query_graphql_field(:merge_request, { iid: merge_request.iid.to_s }, codequality_reports_comparer_fields) + end + + let(:query) { graphql_query_for(:project, { full_path: project.full_path }, merge_request_fields) } + + subject(:result) { graphql_data_at(:project, :merge_request, :codequality_reports_comparer) } + + before do + allow_next_found_instance_of(MergeRequest) do |merge_request| + allow(merge_request).to receive(:compare_codequality_reports).and_return(mock_report) + end + end + + context 'when the user is not authorized to read the field' do + before do + post_graphql(query, current_user: user) + end + + it { is_expected.to be_nil } + end + + context 'when the user is authorized to read the field' do + before_all do + project.add_reporter(user) + end + + before do + post_graphql(query, current_user: user) + end + + context 'when when sast_reports_in_inline_diff FF is disabled' do + before_all do + stub_feature_flags(sast_reports_in_inline_diff: false) + end + + it 'returns null for codequality_reports_comparer field' do + expect(result).to be_nil + end + end + + it 'returns expected data' do + expect(result).to match( + a_hash_including( + { + report: { + status: 'FAILED', + newErrors: [ + { + description: 'Method `new_array` has 12 arguments (exceeds 4 allowed). Consider refactoring.', + fingerprint: '15cdb5c53afd42bc22f8ca366a08d547', + severity: 'MAJOR', + filePath: 'foo.rb', + line: 10, + webUrl: nil, + engineName: 'structure' + }, + { + description: 'Method `backwards_array` has 12 arguments (exceeds 4 allowed). Consider refactoring.', + fingerprint: 'f3bdc1e8c102ba5fbd9e7f6cda51c95e', + severity: 'MAJOR', + filePath: 'foo.rb', + line: 14, + webUrl: nil, + engineName: 'structure' + }, + { + description: 'Avoid parameter lists longer than 5 parameters. [12/5]', + fingerprint: 'ab5f8b935886b942d621399f5a2ca16e', + severity: 'MINOR', + filePath: 'foo.rb', + line: 14, + webUrl: nil, + engineName: 'rubocop' + } + ], + resolvedErrors: [], + existingErrors: [], + summary: { + errored: 3, + resolved: 0, + total: 3 + } + } + }.deep_stringify_keys + ) + ) + end + end +end diff --git a/spec/requests/api/graphql/project/merge_request_spec.rb b/spec/requests/api/graphql/project/merge_request_spec.rb index 80c7258c05d..28aadf850f0 100644 --- a/spec/requests/api/graphql/project/merge_request_spec.rb +++ b/spec/requests/api/graphql/project/merge_request_spec.rb @@ -22,7 +22,8 @@ RSpec.describe 'getting merge request information nested in a project', feature_ it_behaves_like 'a working graphql query' do # we exclude Project.pipeline because it needs arguments - let(:mr_fields) { all_graphql_fields_for('MergeRequest', excluded: %w[jobs pipeline]) } + # and codequalityReportsComparer because no pipeline exist yet + let(:mr_fields) { all_graphql_fields_for('MergeRequest', excluded: %w[jobs pipeline codequalityReportsComparer]) } before do post_graphql(query, current_user: current_user) diff --git a/spec/serializers/codequality_degradation_entity_spec.rb b/spec/serializers/codequality_degradation_entity_spec.rb index 32269e5475b..3d07564c5dc 100644 --- a/spec/serializers/codequality_degradation_entity_spec.rb +++ b/spec/serializers/codequality_degradation_entity_spec.rb @@ -2,18 +2,31 @@ require 'spec_helper' -RSpec.describe CodequalityDegradationEntity do +RSpec.describe CodequalityDegradationEntity, feature_category: :code_quality do let(:entity) { described_class.new(codequality_degradation) } describe '#as_json' do subject { entity.as_json } + context 'when sast_reports_in_inline_diff is disabled' do + before do + stub_feature_flags(sast_reports_in_inline_diff: false) + end + + let(:codequality_degradation) { build(:codequality_degradation_1) } + + it 'does not contain fingerprint' do + expect(subject[:fingerprint]).to be_nil + end + end + context 'when codequality contains an error' do context 'when line is included in location' do let(:codequality_degradation) { build(:codequality_degradation_2) } it 'contains correct codequality degradation details', :aggregate_failures do expect(subject[:description]).to eq("Method `new_array` has 12 arguments (exceeds 4 allowed). Consider refactoring.") + expect(subject[:fingerprint]).to eq("f3bdc1e8c102ba5fbd9e7f6cda51c95e") expect(subject[:severity]).to eq("major") expect(subject[:file_path]).to eq("file_a.rb") expect(subject[:line]).to eq(10) @@ -27,6 +40,7 @@ RSpec.describe CodequalityDegradationEntity do it 'contains correct codequality degradation details', :aggregate_failures do expect(subject[:description]).to eq("Avoid parameter lists longer than 5 parameters. [12/5]") + expect(subject[:fingerprint]).to eq("ab5f8b935886b942d621399f5a2ca16e") expect(subject[:severity]).to eq("minor") expect(subject[:file_path]).to eq("file_b.rb") expect(subject[:line]).to eq(10) @@ -44,6 +58,7 @@ RSpec.describe CodequalityDegradationEntity do it 'lowercases severity', :aggregate_failures do expect(subject[:description]).to eq("Avoid parameter lists longer than 5 parameters. [12/5]") + expect(subject[:fingerprint]).to eq("ab5f8b935886b942d621399f5a2ca16e") expect(subject[:severity]).to eq("minor") expect(subject[:file_path]).to eq("file_b.rb") expect(subject[:line]).to eq(10) diff --git a/spec/serializers/codequality_reports_comparer_serializer_spec.rb b/spec/serializers/codequality_reports_comparer_serializer_spec.rb index 50c8a69737c..7d6f49bf41d 100644 --- a/spec/serializers/codequality_reports_comparer_serializer_spec.rb +++ b/spec/serializers/codequality_reports_comparer_serializer_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' -RSpec.describe CodequalityReportsComparerSerializer do - let(:project) { double(:project) } +RSpec.describe CodequalityReportsComparerSerializer, feature_category: :code_quality do + let(:project) { build_stubbed(:project) } let(:serializer) { described_class.new(project: project).represent(comparer) } let(:comparer) { Gitlab::Ci::Reports::CodequalityReportsComparer.new(base_report, head_report) } let(:base_report) { Gitlab::Ci::Reports::CodequalityReports.new } diff --git a/spec/services/system_notes/issuables_service_spec.rb b/spec/services/system_notes/issuables_service_spec.rb index af660a9b72e..c6d5f3fa385 100644 --- a/spec/services/system_notes/issuables_service_spec.rb +++ b/spec/services/system_notes/issuables_service_spec.rb @@ -28,6 +28,14 @@ RSpec.describe ::SystemNotes::IssuablesService, feature_category: :team_planning expect(subject.note).to eq "marked this issue as related to #{noteable_ref.to_reference(project)}" end end + + context 'with work items' do + let_it_be(:noteable) { create(:work_item, :task, project: project) } + + it 'sets the note text with the correct work item type' do + expect(subject.note).to eq "marked this task as related to #{noteable_ref.to_reference(project)}" + end + end end describe '#unrelate_issuable' do diff --git a/spec/support/database/auto_explain.rb b/spec/support/database/auto_explain.rb index 108d88e37b9..799457034a1 100644 --- a/spec/support/database/auto_explain.rb +++ b/spec/support/database/auto_explain.rb @@ -115,11 +115,16 @@ module AutoExplain private def record_auto_explain?(connection) - ENV['CI'] \ - && ENV['CI_MERGE_REQUEST_LABELS']&.include?('pipeline:record-queries') \ - && ENV['CI_JOB_NAME_SLUG'] != 'db-migrate-non-superuser' \ - && connection.database_version.to_s[0..1].to_i >= 14 \ - && connection.select_one('SHOW is_superuser')['is_superuser'] == 'on' + return false unless ENV['CI'] + return false if ENV['CI_JOB_NAME_SLUG'] == 'db-migrate-non-superuser' + return false if connection.database_version.to_s[0..1].to_i < 14 + return false if connection.select_one('SHOW is_superuser')['is_superuser'] != 'on' + + # This condition matches the pipeline rules for if-merge-request-labels-record-queries + return true if ENV['CI_MERGE_REQUEST_LABELS']&.include?('pipeline:record-queries') + + # This condition matches the pipeline rules for if-default-branch-refs + ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH'] && !ENV['CI_MERGE_REQUEST_IID'] end end end |