diff options
33 files changed, 441 insertions, 157 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57c6d7edc56..2fc78dd28f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -175,6 +175,7 @@ variables: RSPEC_PROFILING_FOLDER_PATH: rspec/profiling RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json RSPEC_VIEWS_INCLUDING_PARTIALS_PATH: rspec/views_including_partials.txt + RSPEC_AUTO_EXPLAIN_LOG_PATH: auto_explain/auto_explain.ndjson.gz TMP_TEST_FOLDER: "${CI_PROJECT_DIR}/tmp/tests" TMP_TEST_GITLAB_WORKHORSE_PATH: "${TMP_TEST_FOLDER}/${GITLAB_WORKHORSE_FOLDER}" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index c501d930352..86c8ee17328 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -213,6 +213,12 @@ - *node-modules-cache # We don't push this cache as it's already rebuilt by `update-assets-compile-*-cache` - *storybook-node-modules-cache-push +.redis-services: + services: + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml + - name: redis:${REDIS_VERSION}-alpine + .pg-base-variables: variables: POSTGRES_HOST_AUTH_METHOD: trust @@ -222,179 +228,226 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-${PG_VERSION}-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off", "-c", "max_locks_per_transaction=256"] alias: postgres - - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 - alias: rediscluster # configure connections in config/redis.yml + - !reference [.redis-services, services] -.db-services-with-redis-6: +.db-services-with-auto-explain: services: - - !reference [.db-services, services] - - name: redis:6.2-alpine + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-${PG_VERSION}-pgvector-0.4.1 + command: + - postgres + - -c + - fsync=off + - -c + - synchronous_commit=off + - -c + - full_page_writes=off + - -c + - max_locks_per_transaction=256 + - -c + - log_filename=pglog + - -c + - log_destination=csvlog + - -c + - logging_collector=true + - -c + - auto_explain.log_min_duration=0 + - -c + - auto_explain.log_format=json + - -c + - auto_explain.log_timing=off + alias: postgres + - !reference [.redis-services, services] -.db-services-with-redis-7: +.zoekt-variables: + variables: + ZOEKT_INDEX_BASE_URL: http://zoekt-ci-image:6060 + ZOEKT_SEARCH_BASE_URL: http://zoekt-ci-image:6070 + +.zoekt-services: services: - - !reference [.db-services, services] - - name: redis:7.0-alpine + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1 + alias: zoekt-ci-image .use-pg12: extends: - .pg-base-variables - - .db-services-with-redis-6 + services: + - !reference [.db-services, services] variables: PG_VERSION: "12" + REDIS_VERSION: "6.2" .use-pg13: extends: - .pg-base-variables - - .db-services-with-redis-6 + services: + - !reference [.db-services, services] variables: PG_VERSION: "13" + REDIS_VERSION: "6.2" .use-pg14: extends: - .pg-base-variables - - .db-services-with-redis-6 + services: + - !reference [.db-services-with-auto-explain, services] variables: PG_VERSION: "14" + REDIS_VERSION: "6.2" .use-pg15: extends: - .pg-base-variables - - .db-services-with-redis-7 + services: + - !reference [.db-services-with-auto-explain, services] variables: PG_VERSION: "15" + REDIS_VERSION: "7.0" -.zoekt-variables: - variables: - ZOEKT_INDEX_BASE_URL: http://zoekt-ci-image:6060 - ZOEKT_SEARCH_BASE_URL: http://zoekt-ci-image:6070 - -.zoekt-services: - services: - - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.1 - alias: zoekt-ci-image - -.es7-base: - extends: - - .pg-base-variables - - .zoekt-variables +.es7-services: services: - - !reference [.db-services-with-redis-6, services] - !reference [.zoekt-services, services] - name: elasticsearch:7.17.6 command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] .use-pg12-es7-ee: - extends: .es7-base - variables: - PG_VERSION: "12" + extends: + - .use-pg12 + - .zoekt-variables + services: + - !reference [.db-services, services] + - !reference [.es7-services, services] .use-pg13-es7-ee: - extends: .es7-base - variables: - PG_VERSION: "13" + extends: + - .use-pg13 + - .zoekt-variables + services: + - !reference [.db-services, services] + - !reference [.es7-services, services] .use-pg14-es7-ee: - extends: .es7-base - variables: - PG_VERSION: "14" + extends: + - .use-pg14 + - .zoekt-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.es7-services, services] .use-pg15-es7-ee: - extends: .es7-base - variables: - PG_VERSION: "15" - -.es8-base: extends: - - .pg-base-variables + - .use-pg15 - .zoekt-variables services: - - !reference [.db-services-with-redis-6, services] + - !reference [.db-services-with-auto-explain, services] + - !reference [.es7-services, services] + +.es8-services: + services: - !reference [.zoekt-services, services] - name: elasticsearch:8.6.2 + +.es8-variables: variables: ES_SETTING_DISCOVERY_TYPE: "single-node" ES_SETTING_XPACK_SECURITY_ENABLED: "false" .use-pg13-es8-ee: - extends: .es8-base - variables: - PG_VERSION: "13" + extends: + - .use-pg13 + - .zoekt-variables + - .es8-variables + services: + - !reference [.db-services, services] + - !reference [.es8-services, services] .use-pg14-es8-ee: - extends: .es8-base - variables: - PG_VERSION: "14" + extends: + - .use-pg14 + - .zoekt-variables + - .es8-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.es8-services, services] .use-pg15-es8-ee: - extends: .es8-base - variables: - PG_VERSION: "15" - -.os1-base: extends: - - .pg-base-variables + - .use-pg15 - .zoekt-variables + - .es8-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.es8-services, services] + +.os1-services: services: - - !reference [.db-services-with-redis-6, services] - !reference [.zoekt-services, services] - name: opensearchproject/opensearch:1.3.5 alias: elasticsearch command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"] .use-pg13-opensearch1-ee: - extends: .os1-base - variables: - PG_VERSION: "13" + extends: + - .use-pg13 + - .zoekt-variables + services: + - !reference [.db-services, services] + - !reference [.os1-services, services] .use-pg14-opensearch1-ee: - extends: .os1-base - variables: - PG_VERSION: "14" + extends: + - .use-pg14 + - .zoekt-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.os1-services, services] .use-pg15-opensearch1-ee: - extends: .os1-base - variables: - PG_VERSION: "15" - -.os2-base: extends: - - .pg-base-variables + - .use-pg15 - .zoekt-variables services: - - !reference [.db-services-with-redis-6, services] + - !reference [.db-services-with-auto-explain, services] + - !reference [.os1-services, services] + +.os2-services: + services: - !reference [.zoekt-services, services] - name: opensearchproject/opensearch:2.2.1 alias: elasticsearch command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"] .use-pg13-opensearch2-ee: - extends: .os2-base - variables: - PG_VERSION: "13" + extends: + - .use-pg13 + - .zoekt-variables + services: + - !reference [.db-services, services] + - !reference [.os2-services, services] .use-pg14-opensearch2-ee: - extends: .os2-base - variables: - PG_VERSION: "14" + extends: + - .use-pg14 + - .zoekt-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.os2-services, services] .use-pg15-opensearch2-ee: - extends: .os2-base - variables: - PG_VERSION: "15" + extends: + - .use-pg15 + - .zoekt-variables + services: + - !reference [.db-services-with-auto-explain, services] + - !reference [.os2-services, services] .use-pg14-clickhouse23: + extends: .use-pg14 services: - - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - alias: postgres - - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 - alias: rediscluster # configure connections in config/redis.yml - - name: redis:6.2-alpine + - !reference [.db-services-with-auto-explain, services] - name: clickhouse/clickhouse-server:23-alpine alias: clickhouse variables: - POSTGRES_HOST_AUTH_METHOD: trust - PG_VERSION: "14" CLICKHOUSE_USER: clickhouse CLICKHOUSE_PASSWORD: clickhouse CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index d5b596a3f81..2844669a89d 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -327,8 +327,8 @@ rspec:deprecations: GIT_STRATEGY: none image: alpine:3.17 script: - - mkdir -p coverage deprecations rspec - - ls coverage/ deprecations/ rspec/ + - mkdir -p coverage deprecations rspec auto_explain + - ls coverage/ deprecations/ rspec/ auto_explain/ artifacts: expire_in: 7d when: always @@ -336,6 +336,7 @@ rspec:deprecations: - coverage/ - deprecations/ - rspec/ + - auto_explain/ rspec:artifact-collector unit: extends: @@ -546,6 +547,23 @@ rspec:flaky-tests-report: paths: - rspec/ +rspec:merge-auto-explain-logs: + extends: + - .coverage-base + - .rails:rules:rspec-merge-auto-explain-logs + stage: post-test + needs: !reference ["rspec:coverage", "needs"] + before_script: + - source scripts/utils.sh + - source scripts/rspec_helpers.sh + script: + - merge_auto_explain_logs + artifacts: + name: auto-explain-logs + expire_in: 31d + paths: + - auto_explain/ + # EE/FOSS: default refs (MRs, default branch, schedules) jobs # ####################################################### diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index 9c2b0406f02..8586723a26e 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -96,6 +96,7 @@ include: expire_in: 31d when: always paths: + - auto_explain/ - coverage/ - crystalball/ - deprecations/ diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 295f22258e9..2d9bfaf7f63 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -89,6 +89,9 @@ .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/' +.if-merge-request-labels-record-queries: &if-merge-request-labels-record-queries + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:record-queries/' + .if-merge-request-labels-jh-contribution: &if-merge-request-labels-jh-contribution if: '$CI_MERGE_REQUEST_LABELS =~ /JiHu contribution/' @@ -2024,6 +2027,15 @@ - <<: *if-merge-request-labels-run-on-pg12 - !reference [".rails:rules:default-branch-schedule-nightly--code-backstage-default-rules", rules] +.rails:rules:rspec-merge-auto-explain-logs: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request-labels-pipeline-expedite + when: never + - <<: *if-merge-request-labels-run-all-rspec + - <<: *if-merge-request-labels-record-queries + .rails:rules:default-branch-schedule-nightly--code-backstage-default-rules: rules: - <<: *if-merge-request-labels-pipeline-expedite diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index 0a559874217..f7e29e43ab1 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -140,10 +140,6 @@ RSpec/MissingFeatureCategory: - 'ee/spec/graphql/ee/types/board_list_type_spec.rb' - 'ee/spec/graphql/ee/types/board_type_spec.rb' - 'ee/spec/graphql/ee/types/boards/board_issue_input_type_spec.rb' - - 'ee/spec/graphql/ee/types/branch_protection_type_spec.rb' - - 'ee/spec/graphql/ee/types/branch_protections/merge_access_level_type_spec.rb' - - 'ee/spec/graphql/ee/types/branch_protections/push_access_level_type_spec.rb' - - 'ee/spec/graphql/ee/types/branch_protections/unprotect_access_level_type_spec.rb' - 'ee/spec/graphql/ee/types/ci/pipeline_merge_request_type_enum_spec.rb' - 'ee/spec/graphql/ee/types/clusters/agent_type_spec.rb' - 'ee/spec/graphql/ee/types/compliance_management/compliance_framework_type_spec.rb' @@ -155,7 +151,6 @@ RSpec/MissingFeatureCategory: - 'ee/spec/graphql/ee/types/milestone_type_spec.rb' - 'ee/spec/graphql/ee/types/mutation_type_spec.rb' - 'ee/spec/graphql/ee/types/notes/noteable_interface_spec.rb' - - 'ee/spec/graphql/ee/types/projects/branch_rule_type_spec.rb' - 'ee/spec/graphql/ee/types/projects/service_type_enum_spec.rb' - 'ee/spec/graphql/ee/types/repository/blob_type_spec.rb' - 'ee/spec/graphql/ee/types/todoable_interface_spec.rb' @@ -277,8 +272,6 @@ RSpec/MissingFeatureCategory: - 'ee/spec/graphql/resolvers/vulnerabilities_grade_resolver_spec.rb' - 'ee/spec/graphql/resolvers/vulnerabilities_resolver_spec.rb' - 'ee/spec/graphql/resolvers/vulnerability_severities_count_resolver_spec.rb' - - 'ee/spec/graphql/types/access_levels/group_type_spec.rb' - - 'ee/spec/graphql/types/access_levels/user_type_spec.rb' - 'ee/spec/graphql/types/admin/cloud_licenses/current_license_type_spec.rb' - 'ee/spec/graphql/types/admin/cloud_licenses/license_history_entry_type_spec.rb' - 'ee/spec/graphql/types/admin/cloud_licenses/subscription_future_entry_type_spec.rb' @@ -299,8 +292,6 @@ RSpec/MissingFeatureCategory: - 'ee/spec/graphql/types/boards/epic_list_metadata_type_spec.rb' - 'ee/spec/graphql/types/boards/epic_list_type_spec.rb' - 'ee/spec/graphql/types/boards/epic_user_preferences_type_spec.rb' - - 'ee/spec/graphql/types/branch_rules/approval_project_rule_type_spec.rb' - - 'ee/spec/graphql/types/branch_rules/external_status_check_type_spec.rb' - 'ee/spec/graphql/types/burnup_chart_daily_totals_type_spec.rb' - 'ee/spec/graphql/types/ci/code_coverage_activity_type_spec.rb' - 'ee/spec/graphql/types/ci/code_coverage_summary_spec.rb' @@ -2218,9 +2209,6 @@ RSpec/MissingFeatureCategory: - 'spec/graphql/types/board_list_type_spec.rb' - 'spec/graphql/types/board_type_spec.rb' - 'spec/graphql/types/boards/board_issue_input_type_spec.rb' - - 'spec/graphql/types/branch_protections/merge_access_level_type_spec.rb' - - 'spec/graphql/types/branch_protections/push_access_level_type_spec.rb' - - 'spec/graphql/types/branch_rules/branch_protection_type_spec.rb' - 'spec/graphql/types/branch_type_spec.rb' - 'spec/graphql/types/ci/analytics_type_spec.rb' - 'spec/graphql/types/ci/config/config_type_spec.rb' @@ -2421,7 +2409,6 @@ RSpec/MissingFeatureCategory: - 'spec/graphql/types/project_statistics_type_spec.rb' - 'spec/graphql/types/project_type_spec.rb' - 'spec/graphql/types/projects/base_service_type_spec.rb' - - 'spec/graphql/types/projects/branch_rule_type_spec.rb' - 'spec/graphql/types/projects/jira_project_type_spec.rb' - 'spec/graphql/types/projects/jira_service_type_spec.rb' - 'spec/graphql/types/projects/repository_language_type_spec.rb' diff --git a/app/assets/javascripts/ci/ci_variable_list/components/ci_group_variables.vue b/app/assets/javascripts/ci/ci_variable_list/components/ci_group_variables.vue index 2045b127a82..842d88e1267 100644 --- a/app/assets/javascripts/ci/ci_variable_list/components/ci_group_variables.vue +++ b/app/assets/javascripts/ci/ci_variable_list/components/ci_group_variables.vue @@ -23,15 +23,6 @@ export default { graphqlId() { return convertToGraphQLId(TYPENAME_GROUP, this.groupId); }, - queriesAvailable() { - if (this.glFeatures.ciGroupEnvScopeGraphql) { - return this.$options.queryData; - } - - return { - ciVariables: this.$options.queryData.ciVariables, - }; - }, }, mutationData: { [ADD_MUTATION_ACTION]: addGroupVariable, @@ -59,6 +50,6 @@ export default { entity="group" :full-path="groupPath" :mutation-data="$options.mutationData" - :query-data="queriesAvailable" + :query-data="$options.queryData" /> </template> diff --git a/app/assets/javascripts/diffs/store/actions.js b/app/assets/javascripts/diffs/store/actions.js index 2a557017953..a5336b5c2e7 100644 --- a/app/assets/javascripts/diffs/store/actions.js +++ b/app/assets/javascripts/diffs/store/actions.js @@ -425,7 +425,9 @@ export const showCommentForm = ({ commit }, { lineCode, fileHash }) => { // works. If we focus the comment form on mount and the comment form gets removed and then // added again the page will scroll in unexpected ways setTimeout(() => { - const el = document.querySelector(`[data-line-code="${lineCode}"] textarea`); + const el = document.querySelector( + `[data-line-code="${lineCode}"] textarea, [data-line-code="${lineCode}"] [contenteditable="true"]`, + ); if (!el) return; diff --git a/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/failed_jobs_list.vue b/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/failed_jobs_list.vue index 36687129cdd..9fad2e8dcc7 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/failed_jobs_list.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/failed_jobs_list.vue @@ -19,7 +19,7 @@ export default { GlLoadingIcon, FailedJobDetails, }, - inject: ['fullPath', 'graphqlPath'], + inject: ['graphqlPath'], props: { isPipelineActive: { required: true, @@ -29,6 +29,10 @@ export default { type: Number, required: true, }, + projectPath: { + type: String, + required: true, + }, }, data() { return { @@ -46,7 +50,7 @@ export default { pollInterval: POLL_INTERVAL, variables() { return { - fullPath: this.fullPath, + fullPath: this.projectPath, pipelineIid: this.pipelineIid, }; }, diff --git a/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget.vue b/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget.vue index 5e49c05f47d..6688f372b07 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget.vue @@ -31,6 +31,10 @@ export default { required: true, type: String, }, + projectPath: { + required: true, + type: String, + }, }, data() { return { @@ -100,6 +104,7 @@ export default { v-if="isExpanded" :is-pipeline-active="isPipelineActive" :pipeline-iid="pipelineIid" + :project-path="projectPath" @failed-jobs-count="setFailedJobsCount" /> </gl-collapse> diff --git a/app/assets/javascripts/pipelines/components/pipelines_list/pipelines_table.vue b/app/assets/javascripts/pipelines/components/pipelines_list/pipelines_table.vue index dbb0b443235..41b3558d992 100644 --- a/app/assets/javascripts/pipelines/components/pipelines_list/pipelines_table.vue +++ b/app/assets/javascripts/pipelines/components/pipelines_list/pipelines_table.vue @@ -1,5 +1,6 @@ <script> import { GlTableLite, GlTooltipDirective } from '@gitlab/ui'; +import { cleanLeadingSeparator } from '~/lib/utils/url_utility'; import { s__, __ } from '~/locale'; import Tracking from '~/tracking'; import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; @@ -146,6 +147,9 @@ export default { const downstream = pipeline.triggered; return keepLatestDownstreamPipelines(downstream); }, + getProjectPath(item) { + return cleanLeadingSeparator(item.project.full_path); + }, failedJobsCount(pipeline) { return pipeline?.failed_builds?.length || 0; }, @@ -225,6 +229,7 @@ export default { :is-pipeline-active="item.active" :pipeline-iid="item.iid" :pipeline-path="item.path" + :project-path="getProjectPath(item)" /> </template> </gl-table-lite> diff --git a/app/controllers/groups/settings/ci_cd_controller.rb b/app/controllers/groups/settings/ci_cd_controller.rb index 169caabf9d8..4bbaf92b126 100644 --- a/app/controllers/groups/settings/ci_cd_controller.rb +++ b/app/controllers/groups/settings/ci_cd_controller.rb @@ -14,7 +14,6 @@ module Groups feature_category :continuous_integration before_action do - push_frontend_feature_flag(:ci_group_env_scope_graphql, group) push_frontend_feature_flag(:ci_variables_pages, current_user) end diff --git a/config/feature_flags/development/ci_group_env_scope_graphql.yml b/config/feature_flags/development/ci_group_env_scope_graphql.yml deleted file mode 100644 index 04b080c67d4..00000000000 --- a/config/feature_flags/development/ci_group_env_scope_graphql.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_group_env_scope_graphql -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/124134 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/416385 -milestone: '16.2' -type: development -group: group::pipeline security -default_enabled: false diff --git a/db/post_migrate/20230717091811_add_prepared_at_index_to_merge_requests_sync.rb b/db/post_migrate/20230717091811_add_prepared_at_index_to_merge_requests_sync.rb new file mode 100644 index 00000000000..4f05c5634b1 --- /dev/null +++ b/db/post_migrate/20230717091811_add_prepared_at_index_to_merge_requests_sync.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddPreparedAtIndexToMergeRequestsSync < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + INDEX_NAME = 'index_merge_requests_on_id_and_prepared_at' + + def up + add_concurrent_index :merge_requests, :id, name: INDEX_NAME, where: "prepared_at IS NULL" + end + + def down + remove_concurrent_index_by_name :merge_requests, INDEX_NAME + end +end diff --git a/db/schema_migrations/20230717091811 b/db/schema_migrations/20230717091811 new file mode 100644 index 00000000000..adcbb70cbeb --- /dev/null +++ b/db/schema_migrations/20230717091811 @@ -0,0 +1 @@ +75ffffb20f6821af4b7fc8d6c8d15fc08e00e5a13f48505935b702452f108fd0
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 69d423037f6..b29b10d6919 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -31921,6 +31921,8 @@ CREATE INDEX index_merge_requests_on_description_trigram ON merge_requests USING CREATE INDEX index_merge_requests_on_head_pipeline_id ON merge_requests USING btree (head_pipeline_id); +CREATE INDEX index_merge_requests_on_id_and_prepared_at ON merge_requests USING btree (id) WHERE (prepared_at IS NULL); + CREATE INDEX index_merge_requests_on_latest_merge_request_diff_id ON merge_requests USING btree (latest_merge_request_diff_id); CREATE INDEX index_merge_requests_on_merge_user_id ON merge_requests USING btree (merge_user_id) WHERE (merge_user_id IS NOT NULL); diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 2ce64aca218..d88c56830aa 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -23841,7 +23841,7 @@ Represents a vulnerability. | <a id="vulnerabilitydetails"></a>`details` | [`[VulnerabilityDetail!]!`](#vulnerabilitydetail) | Details of the vulnerability. | | <a id="vulnerabilitydetectedat"></a>`detectedAt` | [`Time!`](#time) | Timestamp of when the vulnerability was first detected. | | <a id="vulnerabilitydiscussions"></a>`discussions` | [`DiscussionConnection!`](#discussionconnection) | All discussions on this noteable. (see [Connections](#connections)) | -| <a id="vulnerabilitydismissalreason"></a>`dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason for dismissal. Returns `null` if `expose_dismissal_reason`feature flag is disabled. | +| <a id="vulnerabilitydismissalreason"></a>`dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason for dismissal. Returns `null` for states other than `dismissed`. Returns `null` if `expose_dismissal_reason` feature flag is disabled. | | <a id="vulnerabilitydismissedat"></a>`dismissedAt` | [`Time`](#time) | Timestamp of when the vulnerability state was changed to dismissed. | | <a id="vulnerabilitydismissedby"></a>`dismissedBy` | [`UserCore`](#usercore) | User that dismissed the vulnerability. | | <a id="vulnerabilityexternalissuelinks"></a>`externalIssueLinks` | [`VulnerabilityExternalIssueLinkConnection!`](#vulnerabilityexternalissuelinkconnection) | List of external issue links related to the vulnerability. (see [Connections](#connections)) | diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index d06b1ababb8..3cad9c9cb3e 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -104,12 +104,14 @@ You can view which pipeline published the package, and the commit and user who t ### To import packages If you already have packages built in a different registry, you can import them -into your GitLab package registry with the [Packages Importer](https://gitlab.com/gitlab-org/ci-cd/package-stage/pkgs_importer) +into your GitLab package registry with the [Packages Importer](https://gitlab.com/gitlab-org/ci-cd/package-stage/pkgs_importer). The Packages Importer runs a CI/CD pipeline that [can import these package types](https://gitlab.com/gitlab-org/ci-cd/package-stage/pkgs_importer#formats-supported): +- Maven - NPM - NuGet +- PyPI ## Reduce storage usage diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index 2a48b66bb5c..49ec564eb8d 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -100,7 +100,7 @@ module ExtractsRef # rubocop:enable Gitlab/ModuleWithInstanceVariables def tree - @tree ||= @repo.tree(@commit.id, @path, ref_type: ref_type) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @tree ||= @repo.tree(@commit.id, @path) # rubocop:disable Gitlab/ModuleWithInstanceVariables end def extract_ref_path diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 600b49e62f1..b768a2214a6 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -23100,6 +23100,9 @@ msgstr "" msgid "IdentityVerification|The code is incorrect. Enter it again, or send a new code." msgstr "" +msgid "IdentityVerification|There was a problem with the credit card details you entered. Use a different credit card and try again." +msgstr "" + msgid "IdentityVerification|Verification code" msgstr "" diff --git a/scripts/merge-auto-explain-logs b/scripts/merge-auto-explain-logs new file mode 100755 index 00000000000..0dff4fb33f8 --- /dev/null +++ b/scripts/merge-auto-explain-logs @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "set" +require "json" + +fingerprints = Set.new + +ARGF.each_line do |line| + fingerprint = JSON.parse(line)['fingerprint'] + $stdout.puts(line) && $stdout.flush if fingerprints.add?(fingerprint) +end + +warn("auto_explain log contains #{fingerprints.size} entries") diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index fd478cbd3ae..f92b10a19ff 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -483,3 +483,13 @@ function is_rspec_last_run_results_file_missing() { return 1 fi } + +function merge_auto_explain_logs() { + local auto_explain_logs_path="$(dirname "${RSPEC_AUTO_EXPLAIN_LOG_PATH}")/" + + for file in ${auto_explain_logs_path}*.gz; do + (gunzip -c "${file}" && rm -f "${file}" || true) + done | \ + scripts/merge-auto-explain-logs | \ + gzip -c > "${RSPEC_AUTO_EXPLAIN_LOG_PATH}" +} diff --git a/spec/controllers/projects/tree_controller_spec.rb b/spec/controllers/projects/tree_controller_spec.rb index ffec670e97d..a409030e359 100644 --- a/spec/controllers/projects/tree_controller_spec.rb +++ b/spec/controllers/projects/tree_controller_spec.rb @@ -90,8 +90,15 @@ RSpec.describe Projects::TreeController, feature_category: :source_code_manageme context 'and explicitly requesting a branch' do let(:ref_type) { 'heads' } + it 'checks for tree with ref_type' do + allow(project.repository).to receive(:tree).and_call_original + expect(project.repository).to receive(:tree).with(id, '', ref_type: 'heads').and_call_original + request + end + it 'finds the branch' do expect(requested_ref_double).not_to receive(:find) + request expect(response).to be_ok end @@ -100,6 +107,12 @@ RSpec.describe Projects::TreeController, feature_category: :source_code_manageme context 'and explicitly requesting a tag' do let(:ref_type) { 'tags' } + it 'checks for tree with ref_type' do + allow(project.repository).to receive(:tree).and_call_original + expect(project.repository).to receive(:tree).with(id, '', ref_type: 'tags').and_call_original + request + end + it 'finds the tag' do expect(requested_ref_double).not_to receive(:find) request @@ -110,7 +123,13 @@ RSpec.describe Projects::TreeController, feature_category: :source_code_manageme end context "valid branch, no path" do - let(:id) { 'master' } + let(:id) { 'flatten-dir' } + + it 'checks for tree without ref_type' do + allow(project.repository).to receive(:tree).and_call_original + expect(project.repository).to receive(:tree).with(RepoHelpers.another_sample_commit.id, '').and_call_original + request + end it 'responds with success' do request diff --git a/spec/frontend/ci/ci_variable_list/components/ci_group_variables_spec.js b/spec/frontend/ci/ci_variable_list/components/ci_group_variables_spec.js index b364f098a3a..567a49d663c 100644 --- a/spec/frontend/ci/ci_variable_list/components/ci_group_variables_spec.js +++ b/spec/frontend/ci/ci_variable_list/components/ci_group_variables_spec.js @@ -30,7 +30,6 @@ describe('Ci Group Variable wrapper', () => { provide: { ...mockProvide, glFeatures: { - ciGroupEnvScopeGraphql: false, groupScopedCiVariables: false, ...featureFlags, }, @@ -61,6 +60,10 @@ describe('Ci Group Variable wrapper', () => { lookup: expect.any(Function), query: getGroupVariables, }, + environments: { + lookup: expect.any(Function), + query: getGroupEnvironments, + }, }, refetchAfterMutation: false, }); @@ -88,26 +91,4 @@ describe('Ci Group Variable wrapper', () => { }); }); }); - - describe('ciGroupEnvScopeGraphql feature flag', () => { - describe('When enabled', () => { - beforeEach(() => { - createComponent({ featureFlags: { ciGroupEnvScopeGraphql: true } }); - }); - - it('Passes down environments query to variable shared component', () => { - expect(findCiShared().props('queryData').environments.query).toBe(getGroupEnvironments); - }); - }); - - describe('When disabled', () => { - beforeEach(() => { - createComponent(); - }); - - it('Does not pass down environments query to variable shared component', () => { - expect(findCiShared().props('queryData').environments).toBe(undefined); - }); - }); - }); }); diff --git a/spec/frontend/pipelines/components/pipelines_list/failure_widget/failed_jobs_list_spec.js b/spec/frontend/pipelines/components/pipelines_list/failure_widget/failed_jobs_list_spec.js index fc8263c6c4d..872a615458c 100644 --- a/spec/frontend/pipelines/components/pipelines_list/failure_widget/failed_jobs_list_spec.js +++ b/spec/frontend/pipelines/components/pipelines_list/failure_widget/failed_jobs_list_spec.js @@ -26,11 +26,10 @@ describe('FailedJobsList component', () => { graphqlResourceEtag: 'api/graphql', isPipelineActive: false, pipelineIid: 1, - pipelinePath: '/pipelines/1', + projectPath: 'namespace/project/', }; const defaultProvide = { - fullPath: 'namespace/project/', graphqlPath: 'api/graphql', }; @@ -65,6 +64,21 @@ describe('FailedJobsList component', () => { mockFailedJobsResponse = jest.fn(); }); + describe('on mount', () => { + beforeEach(() => { + mockFailedJobsResponse.mockResolvedValue(failedJobsMock); + createComponent(); + }); + + it('fires the graphql query', () => { + expect(mockFailedJobsResponse).toHaveBeenCalledTimes(1); + expect(mockFailedJobsResponse).toHaveBeenCalledWith({ + fullPath: defaultProps.projectPath, + pipelineIid: defaultProps.pipelineIid, + }); + }); + }); + describe('when loading failed jobs', () => { beforeEach(() => { mockFailedJobsResponse.mockResolvedValue(failedJobsMock); diff --git a/spec/frontend/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget_spec.js b/spec/frontend/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget_spec.js index c1a885391e9..9a33670d034 100644 --- a/spec/frontend/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget_spec.js +++ b/spec/frontend/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget_spec.js @@ -13,6 +13,7 @@ describe('PipelineFailedJobsWidget component', () => { isPipelineActive: false, pipelineIid: 1, pipelinePath: '/pipelines/1', + projectPath: 'namespace/project/', }; const defaultProvide = { diff --git a/spec/frontend/pipelines/pipelines_table_spec.js b/spec/frontend/pipelines/pipelines_table_spec.js index 251d823cc37..e8a5e256a45 100644 --- a/spec/frontend/pipelines/pipelines_table_spec.js +++ b/spec/frontend/pipelines/pipelines_table_spec.js @@ -5,6 +5,7 @@ import fixture from 'test_fixtures/pipelines/pipelines.json'; import { mockTracking, unmockTracking } from 'helpers/tracking_helper'; import { extendedWrapper } from 'helpers/vue_test_utils_helper'; import PipelineMiniGraph from '~/pipelines/components/pipeline_mini_graph/pipeline_mini_graph.vue'; +import PipelineFailedJobsWidget from '~/pipelines/components/pipelines_list/failure_widget/pipeline_failed_jobs_widget.vue'; import PipelineOperations from '~/pipelines/components/pipelines_list/pipeline_operations.vue'; import PipelineTriggerer from '~/pipelines/components/pipelines_list/pipeline_triggerer.vue'; import PipelineUrl from '~/pipelines/components/pipelines_list/pipeline_url.vue'; @@ -74,6 +75,7 @@ describe('Pipelines Table', () => { const findTimeAgo = () => wrapper.findComponent(PipelinesTimeago); const findActions = () => wrapper.findComponent(PipelineOperations); + const findPipelineFailureWidget = () => wrapper.findComponent(PipelineFailedJobsWidget); const findTableRows = () => wrapper.findAllByTestId('pipeline-table-row'); const findStatusTh = () => wrapper.findByTestId('status-th'); const findPipelineTh = () => wrapper.findByTestId('pipeline-th'); @@ -189,6 +191,7 @@ describe('Pipelines Table', () => { it('does not render', () => { expect(findTableRows()).toHaveLength(1); + expect(findPipelineFailureWidget().exists()).toBe(false); }); }); @@ -197,8 +200,21 @@ describe('Pipelines Table', () => { beforeEach(() => { createComponent({ pipelines: [pipeline] }, provideWithDetails); }); + it('renders', () => { expect(findTableRows()).toHaveLength(2); + expect(findPipelineFailureWidget().exists()).toBe(true); + }); + + it('passes the expected props', () => { + expect(findPipelineFailureWidget().props()).toStrictEqual({ + failedJobsCount: pipeline.failed_builds.length, + isPipelineActive: pipeline.active, + pipelineIid: pipeline.iid, + pipelinePath: pipeline.path, + // Make sure the forward slash was removed + projectPath: 'frontend-fixtures/pipelines-project', + }); }); }); @@ -212,6 +228,7 @@ describe('Pipelines Table', () => { it('does not render', () => { expect(findTableRows()).toHaveLength(1); + expect(findPipelineFailureWidget().exists()).toBe(false); }); }); }); diff --git a/spec/graphql/types/branch_protections/merge_access_level_type_spec.rb b/spec/graphql/types/branch_protections/merge_access_level_type_spec.rb index 8cc1005d97e..0586a643196 100644 --- a/spec/graphql/types/branch_protections/merge_access_level_type_spec.rb +++ b/spec/graphql/types/branch_protections/merge_access_level_type_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe GitlabSchema.types['MergeAccessLevel'] do +RSpec.describe GitlabSchema.types['MergeAccessLevel'], feature_category: :source_code_management do subject { described_class } let(:fields) { %i[access_level access_level_description] } diff --git a/spec/graphql/types/branch_protections/push_access_level_type_spec.rb b/spec/graphql/types/branch_protections/push_access_level_type_spec.rb index c78c0bda74c..70944bac9bd 100644 --- a/spec/graphql/types/branch_protections/push_access_level_type_spec.rb +++ b/spec/graphql/types/branch_protections/push_access_level_type_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe GitlabSchema.types['PushAccessLevel'] do +RSpec.describe GitlabSchema.types['PushAccessLevel'], feature_category: :source_code_management do subject { described_class } let(:fields) { %i[access_level access_level_description] } diff --git a/spec/graphql/types/branch_rules/branch_protection_type_spec.rb b/spec/graphql/types/branch_rules/branch_protection_type_spec.rb index bbc92fd8fef..d74c76d3f94 100644 --- a/spec/graphql/types/branch_rules/branch_protection_type_spec.rb +++ b/spec/graphql/types/branch_rules/branch_protection_type_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe GitlabSchema.types['BranchProtection'] do +RSpec.describe GitlabSchema.types['BranchProtection'], feature_category: :source_code_management do subject { described_class } let(:fields) { %i[merge_access_levels push_access_levels allow_force_push] } diff --git a/spec/graphql/types/projects/branch_rule_type_spec.rb b/spec/graphql/types/projects/branch_rule_type_spec.rb index 54ea4f6857b..fc7bf4252f1 100644 --- a/spec/graphql/types/projects/branch_rule_type_spec.rb +++ b/spec/graphql/types/projects/branch_rule_type_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe GitlabSchema.types['BranchRule'] do +RSpec.describe GitlabSchema.types['BranchRule'], feature_category: :source_code_management do include GraphqlHelpers subject { described_class } diff --git a/spec/models/concerns/where_composite_spec.rb b/spec/models/concerns/where_composite_spec.rb index 6abdd12aac5..b2b9602f5ee 100644 --- a/spec/models/concerns/where_composite_spec.rb +++ b/spec/models/concerns/where_composite_spec.rb @@ -4,7 +4,7 @@ require 'spec_helper' RSpec.describe WhereComposite do describe '.where_composite' do - let_it_be(:test_table_name) { "_test_table_#{SecureRandom.hex(10)}" } + let_it_be(:test_table_name) { "_test_table_where_composite" } let(:model) do tbl_name = test_table_name diff --git a/spec/support/database/auto_explain.rb b/spec/support/database/auto_explain.rb new file mode 100644 index 00000000000..108d88e37b9 --- /dev/null +++ b/spec/support/database/auto_explain.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module AutoExplain + class << self + def setup + Gitlab::Database::EachDatabase.each_connection do |connection| + next unless record_auto_explain?(connection) + + connection.execute("LOAD 'auto_explain'") + + # This param can only be set on pg14+ so we can't set it when starting postgres. + connection.execute('ALTER SYSTEM SET compute_query_id TO on') + connection.execute('SELECT pg_reload_conf()') + end + end + + def record + Gitlab::Database::EachDatabase.each_connection do |connection, connection_name| + next unless record_auto_explain?(connection) + + connection.execute(<<~SQL.squish) + CREATE EXTENSION IF NOT EXISTS file_fdw; + CREATE SERVER IF NOT EXISTS pglog FOREIGN DATA WRAPPER file_fdw; + SQL + + csvlog_columns = [ + 'log_time timestamp(3) with time zone', + 'user_name text', + 'database_name text', + 'process_id integer', + 'connection_from text', + 'session_id text', + 'session_line_num bigint', + 'command_tag text', + 'session_start_time timestamp with time zone', + 'virtual_transaction_id text', + 'transaction_id bigint', + 'error_severity text', + 'sql_state_code text', + 'message text', + 'detail text', + 'hint text', + 'internal_query text', + 'internal_query_pos integer', + 'context text', + 'query text', + 'query_pos integer', + 'location text', + 'application_name text', + 'backend_type text', + 'leader_pid integer', + 'query_id bigint' + ] + + connection.transaction do + connection.execute(<<~SQL.squish) + CREATE FOREIGN TABLE IF NOT EXISTS pglog (#{csvlog_columns.join(', ')}) + SERVER pglog + OPTIONS ( filename 'log/pglog.csv', format 'csv' ); + SQL + + log_file = Rails.root.join( + File.dirname(ENV.fetch('RSPEC_AUTO_EXPLAIN_LOG_PATH', 'auto_explain/auto_explain.ndjson.gz')), + "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.#{Process.pid}.#{connection_name}.ndjson.gz" + ) + + FileUtils.mkdir_p(File.dirname(log_file)) + + fingerprints = Set.new + recording_start = Time.now + + Zlib::GzipWriter.open(log_file) do |gz| + pg = connection.raw_connection + + pg.exec('SET statement_timeout TO 0;') + + pg.send_query(<<~SQL.squish) + SELECT DISTINCT ON (m.query_id) + m.message->>'Query Text' as query, m.message->'Plan' as plan + FROM ( + SELECT substring(message from '\{.*$')::jsonb AS message, query_id + FROM pglog + WHERE message LIKE '%{%' + ) m + ORDER BY m.query_id; + SQL + + pg.set_single_row_mode + pg.get_result.stream_each do |row| + query = row['query'] + fingerprint = PgQuery.fingerprint(query) + next unless fingerprints.add?(fingerprint) + + plan = Gitlab::Json.parse(row['plan']) + + output = { + query: query, + plan: plan, + fingerprint: fingerprint, + normalized: PgQuery.normalize(query) + } + + gz.puts Gitlab::Json.generate(output) + end + + puts "auto_explain log contains #{fingerprints.size} entries for #{connection_name}, writing to #{log_file}" + puts "took #{Time.now - recording_start}" + end + + raise ActiveRecord::Rollback + end + end + end + + private + + def record_auto_explain?(connection) + ENV['CI'] \ + && ENV['CI_MERGE_REQUEST_LABELS']&.include?('pipeline:record-queries') \ + && ENV['CI_JOB_NAME_SLUG'] != 'db-migrate-non-superuser' \ + && connection.database_version.to_s[0..1].to_i >= 14 \ + && connection.select_one('SHOW is_superuser')['is_superuser'] == 'on' + end + end +end + +RSpec.configure do |config| + config.before(:suite) do + AutoExplain.setup + end + + config.after(:suite) do + AutoExplain.record + end +end |