diff options
70 files changed, 540 insertions, 260 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2b918e35a0..a5c57b35f08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,7 @@ default: OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY2" .default-branch-pipeline-failure-variables: &default-branch-pipeline-failure-variables - CREATE_ISSUES_FOR_FAILING_TESTS: "true" + CREATE_RAILS_TEST_FAILURE_ISSUES: "true" workflow: name: '$PIPELINE_NAME' diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index 1afc4eb8c97..b8f51a2540d 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -3,31 +3,6 @@ dependencies: [] cache: {} -create-issues-for-failing-tests: - extends: - - .notify-defaults - - .notify:rules:create-issues-for-failing-tests - image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION} - variables: - FAILED_TESTS_DIR: "${CI_PROJECT_DIR}/tmp/failed_tests" - FAILING_ISSUES_PROJECT: "gitlab-org/quality/engineering-productivity/flaky-tests-playground" - FAILING_ISSUE_JSON_DIR: "${CI_PROJECT_DIR}/tmp/issues" - before_script: - - source ./scripts/utils.sh - - source ./scripts/rspec_helpers.sh - - install_gitlab_gem - script: - - mkdir -p "${FAILING_ISSUE_JSON_DIR}" - - retrieve_failed_tests "${FAILED_TESTS_DIR}" "json" "latest" - - scripts/pipeline/create_test_failure_issues.rb --project "${FAILING_ISSUES_PROJECT}" --tests-report-file "${FAILED_TESTS_DIR}/rspec_failed_tests.json" --issues-json-folder "${FAILING_ISSUE_JSON_DIR}" --api-token "${FAILING_ISSUES_PROJECT_TOKEN}" - - scripts/pipeline/create_test_failure_issues.rb --project "${FAILING_ISSUES_PROJECT}" --tests-report-file "${FAILED_TESTS_DIR}/rspec_ee_failed_tests.json" --issues-json-folder "${FAILING_ISSUE_JSON_DIR}" --api-token "${FAILING_ISSUES_PROJECT_TOKEN}" - artifacts: - paths: - - ${FAILED_TESTS_DIR}/ - - ${FAILING_ISSUE_JSON_DIR}/ - when: always - expire_in: 2 days - notify-package-and-test-failure: extends: - .notify-defaults diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index c0ea7d7078a..8ca99edf9bf 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -75,6 +75,14 @@ include: # and they should run on their own jobs so we don't need to run them # in unit tests again. - rspec_paralellized_job "--tag ~quarantine --tag ~level:background_migration" + after_script: + - echo -e "\e[0Ksection_start:`date +%s`:report_results_section[collapsed=true]\r\e[0KReport results" + - | + if [ "$CREATE_RAILS_TEST_FAILURE_ISSUES" == "true" ]; then + bundle exec relate-failure-issue --input-files "rspec/rspec-*.json" --system-log-files "log" --project "gitlab-org-sandbox/rails-test-failures" --token "${RAILS_TEST_FAILURES_PROJECT_TOKEN}"; + fi + - echo -e "\e[0Ksection_end:`date +%s`:report_results_section\r\e[0K" + allow_failure: exit_codes: !reference [.rspec-base, variables, SUCCESSFULLY_RETRIED_TEST_EXIT_CODE] @@ -98,6 +106,8 @@ include: script: - !reference [.base-script, script] - rspec_paralellized_job "--tag ~quarantine --tag ~zoekt" + after_script: + - !reference [.rspec-base, after_script] .rspec-base-pg12: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index dfb41bdaa74..6368dd33542 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -1354,17 +1354,6 @@ ########## # Notify # ########## -.notify:rules:create-issues-for-failing-tests: - rules: - - if: '$CREATE_ISSUES_FOR_FAILING_TESTS != "true"' - when: never - # Don't report child pipeline failures - - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' - when: never - - <<: *if-dot-com-gitlab-org-default-branch - when: on_failure - allow_failure: true - .notify:rules:notify-package-and-test-failure: rules: - <<: *if-not-canonical-namespace diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index d89bd63f791..d197a34c87b 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -eefa5b7efbc45ea1d3569beaa23ea960f741fa54 +671e8fd3137e63ef507e701cb86d232aa893fde0 @@ -473,6 +473,8 @@ group :test do # Moved in `test` because https://gitlab.com/gitlab-org/gitlab/-/issues/217527 gem 'derailed_benchmarks', require: false + + gem 'gitlab_quality-test_tooling', '~> 0.8.0', require: false end gem 'octokit', '~> 4.15' diff --git a/Gemfile.checksum b/Gemfile.checksum index 3870f84f626..a0cc86f4a01 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -218,6 +218,7 @@ {"name":"gitlab-styles","version":"10.0.0","platform":"ruby","checksum":"8a1b20f7b5f351605ff4ed4ec648ef37226f2774d1e1377ed99389448d6913f0"}, {"name":"gitlab_chronic_duration","version":"0.10.6.2","platform":"ruby","checksum":"6dda4cfe7dca9b958f163ac8835c3d9cc70cf8df8cbb89bb2fbf9ba4375105fb"}, {"name":"gitlab_omniauth-ldap","version":"2.2.0","platform":"ruby","checksum":"bb4d20acb3b123ed654a8f6a47d3fac673ece7ed0b6992edb92dca14bad2838c"}, +{"name":"gitlab_quality-test_tooling","version":"0.8.0","platform":"ruby","checksum":"e1972749dd23979abff5960628138c37ab35e40462c581d08c105c9ecf54edf0"}, {"name":"globalid","version":"1.1.0","platform":"ruby","checksum":"b337e1746f0c8cb0a6c918234b03a1ddeb4966206ce288fbb57779f59b2d154f"}, {"name":"gon","version":"6.4.0","platform":"ruby","checksum":"e3a618d659392890f1aa7db420f17c75fd7d35aeb5f8fe003697d02c4b88d2f0"}, {"name":"google-apis-androidpublisher_v3","version":"0.34.0","platform":"ruby","checksum":"d7e1d7dd92f79c498fe2082222a1740d788e022e660c135564b3fd299cab5425"}, @@ -617,6 +618,7 @@ {"name":"sync","version":"0.5.0","platform":"ruby","checksum":"668356cc07c59ac7ed9ecf34fec3929831f179c07adb1f3e1c3b7a1609a638fd"}, {"name":"sys-filesystem","version":"1.4.3","platform":"ruby","checksum":"390919de89822ad6d3ba3daf694d720be9d83ed95cdf7adf54d4573c98b17421"}, {"name":"sysexits","version":"1.2.0","platform":"ruby","checksum":"598241c4ae57baa403c125182dfdcc0d1ac4c0fb606dd47fbed57e4aaf795662"}, +{"name":"table_print","version":"1.5.7","platform":"ruby","checksum":"436664281f93387b882335795e16cfeeb839ad0c785ff7f9110fc0f17c68b5cb"}, {"name":"tanuki_emoji","version":"0.6.0","platform":"ruby","checksum":"4ce91aefed2d076b73fba3eff50e89660c3d25691787a9fe4c0dfabb4218c12a"}, {"name":"telesign","version":"2.2.4","platform":"ruby","checksum":"dcc6e96ea7bcb4da1e2ae786bfe7a4d670a4b5f94ae95dfcdde77d547c544c42"}, {"name":"telesignenterprise","version":"2.2.2","platform":"ruby","checksum":"f147a03263a8c2fe0a0db1a7a9454a6ee37d9e8abd58eaca305bdd8081f9f1b3"}, diff --git a/Gemfile.lock b/Gemfile.lock index aa7927ef987..9eab52f40a3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -624,6 +624,15 @@ GEM omniauth (>= 1.3, < 3) pyu-ruby-sasl (>= 0.0.3.3, < 0.1) rubyntlm (~> 0.5) + gitlab_quality-test_tooling (0.8.0) + activesupport (~> 6.1) + gitlab (~> 4.19) + http (~> 5.0) + nokogiri (~> 1.10) + parallel (>= 1, < 2) + rainbow (>= 3, < 4) + table_print (= 1.5.7) + zeitwerk (>= 2, < 3) globalid (1.1.0) activesupport (>= 5.0) gon (6.4.0) @@ -1499,6 +1508,7 @@ GEM sys-filesystem (1.4.3) ffi (~> 1.1) sysexits (1.2.0) + table_print (1.5.7) tanuki_emoji (0.6.0) telesign (2.2.4) net-http-persistent (>= 3.0.0, < 5.0) @@ -1752,6 +1762,7 @@ DEPENDENCIES gitlab-styles (~> 10.0.0) gitlab_chronic_duration (~> 0.10.6.2) gitlab_omniauth-ldap (~> 2.2.0) + gitlab_quality-test_tooling (~> 0.8.0) gon (~> 6.4.0) google-apis-androidpublisher_v3 (~> 0.34.0) google-apis-cloudbilling_v1 (~> 0.21.0) diff --git a/app/assets/javascripts/batch_comments/stores/modules/batch_comments/getters.js b/app/assets/javascripts/batch_comments/stores/modules/batch_comments/getters.js index 75e4ae63c18..28b9100c5f3 100644 --- a/app/assets/javascripts/batch_comments/stores/modules/batch_comments/getters.js +++ b/app/assets/javascripts/batch_comments/stores/modules/batch_comments/getters.js @@ -71,7 +71,7 @@ export const draftsForLine = (state, getters) => (diffFileSha, line, side = null const showDraftsForThisSide = showDraftOnSide(line, side); if (showDraftsForThisSide && draftsForFile?.[key]) { - return draftsForFile[key]; + return draftsForFile[key].filter((d) => d.position.position_type === 'text'); } return []; }; diff --git a/app/assets/javascripts/diffs/components/diff_file.vue b/app/assets/javascripts/diffs/components/diff_file.vue index 4ffdcd78422..8e1c6cecbd1 100644 --- a/app/assets/javascripts/diffs/components/diff_file.vue +++ b/app/assets/javascripts/diffs/components/diff_file.vue @@ -465,18 +465,18 @@ export default { </gl-alert> <div v-if="showFileDiscussions" class="gl-border-b" data-testid="file-discussions"> <div class="diff-file-discussions-wrapper"> - <diff-file-drafts - :file-hash="file.file_hash" - :show-pin="false" - :position-type="$options.FILE_DIFF_POSITION_TYPE" - class="diff-file-discussions" - /> <diff-discussions v-if="fileDiscussions.length" class="diff-file-discussions" data-testid="diff-file-discussions" :discussions="fileDiscussions" /> + <diff-file-drafts + :file-hash="file.file_hash" + :show-pin="false" + :position-type="$options.FILE_DIFF_POSITION_TYPE" + class="diff-file-discussions" + /> <note-form v-if="file.hasCommentForm" :save-button-title="__('Comment')" diff --git a/app/assets/javascripts/pages/groups/new/components/app.vue b/app/assets/javascripts/pages/groups/new/components/app.vue index 167f56bbfcf..3ee15077d00 100644 --- a/app/assets/javascripts/pages/groups/new/components/app.vue +++ b/app/assets/javascripts/pages/groups/new/components/app.vue @@ -39,6 +39,11 @@ export default { required: false, default: false, }, + isSaas: { + type: Boolean, + required: false, + default: false, + }, }, computed: { initialBreadcrumbs() { @@ -93,6 +98,7 @@ export default { :initial-breadcrumbs="initialBreadcrumbs" :panels="panels" :title="s__('GroupsNew|Create new group')" + :is-saas="isSaas" persistence-key="new_group_last_active_tab" /> </template> diff --git a/app/assets/javascripts/pages/groups/new/index.js b/app/assets/javascripts/pages/groups/new/index.js index 2e53324717c..84e031ae67a 100644 --- a/app/assets/javascripts/pages/groups/new/index.js +++ b/app/assets/javascripts/pages/groups/new/index.js @@ -27,6 +27,7 @@ function initNewGroupCreation(el) { parentGroupUrl, parentGroupName, importExistingGroupPath, + isSaas, } = el.dataset; const props = { @@ -36,6 +37,7 @@ function initNewGroupCreation(el) { parentGroupName, importExistingGroupPath, hasErrors: parseBoolean(hasErrors), + isSaas: parseBoolean(isSaas), }; const apolloProvider = new VueApollo({ diff --git a/app/assets/javascripts/sentry/index.js b/app/assets/javascripts/sentry/index.js index ea835945aa9..cf6a79fe939 100644 --- a/app/assets/javascripts/sentry/index.js +++ b/app/assets/javascripts/sentry/index.js @@ -13,7 +13,7 @@ const index = function index() { process.env.NODE_ENV === 'production' ? [gon.gitlab_url] : [gon.gitlab_url, 'webpack-internal://'], - release: gon.revision, + release: gon?.version, tags: { revision: gon?.revision, feature_category: gon?.feature_category, diff --git a/app/assets/javascripts/vue_shared/new_namespace/new_namespace_page.vue b/app/assets/javascripts/vue_shared/new_namespace/new_namespace_page.vue index c8c7deff882..5ab2e346a7a 100644 --- a/app/assets/javascripts/vue_shared/new_namespace/new_namespace_page.vue +++ b/app/assets/javascripts/vue_shared/new_namespace/new_namespace_page.vue @@ -43,6 +43,11 @@ export default { type: String, required: true, }, + isSaas: { + type: Boolean, + required: false, + default: false, + }, }, data() { @@ -81,11 +86,7 @@ export default { }, showNewTopLevelGroupAlert() { - if (this.activePanel.detailProps === undefined) { - return false; - } - - return this.activePanel.detailProps.parentGroupName === ''; + return this.isSaas && this.activePanel.detailProps?.parentGroupName === ''; }, showSuperSidebarToggle() { diff --git a/app/controllers/graphql_controller.rb b/app/controllers/graphql_controller.rb index 0d348624864..3d3b7f31dfd 100644 --- a/app/controllers/graphql_controller.rb +++ b/app/controllers/graphql_controller.rb @@ -276,7 +276,8 @@ class GraphqlController < ApplicationController def execute_introspection_query if introspection_query_can_use_cache? - log_introspection_query_message(true) + Gitlab::AppLogger.info(message: "IntrospectionQueryCache hit") + log_introspection_query_cache_details(true) # Context for caching: https://gitlab.com/gitlab-org/gitlab/-/issues/409448 Rails.cache.fetch( @@ -285,7 +286,8 @@ class GraphqlController < ApplicationController execute_query.to_json end else - log_introspection_query_message(false) + Gitlab::AppLogger.info(message: "IntrospectionQueryCache miss") + log_introspection_query_cache_details(false) execute_query end @@ -304,13 +306,13 @@ class GraphqlController < ApplicationController ['introspection-query-cache', Gitlab.revision, context[:remove_deprecated]] end - def log_introspection_query_message(can_use_introspection_query_cache) + def log_introspection_query_cache_details(can_use_introspection_query_cache) Gitlab::AppLogger.info( message: "IntrospectionQueryCache", - can_use_introspection_query_cache: can_use_introspection_query_cache, + can_use_introspection_query_cache: can_use_introspection_query_cache.to_s, query: query, - variables: build_variables(params[:variables]), - introspection_query_cache_key: introspection_query_cache_key + variables: build_variables(params[:variables]).to_s, + introspection_query_cache_key: introspection_query_cache_key.to_s ) end end diff --git a/app/helpers/groups_helper.rb b/app/helpers/groups_helper.rb index 2ced1bec5e9..a4f463a23be 100644 --- a/app/helpers/groups_helper.rb +++ b/app/helpers/groups_helper.rb @@ -85,6 +85,7 @@ module GroupsHelper end end + # Overridden in EE def remove_group_message(group) _("You are going to remove %{group_name}. This will also delete all of its subgroups and projects. Removed groups CANNOT be restored! Are you ABSOLUTELY sure?") % { group_name: group.name } @@ -128,7 +129,8 @@ module GroupsHelper { parent_group_url: group.parent && group_url(group.parent), parent_group_name: group.parent&.name, - import_existing_group_path: new_group_path(parent_id: group.parent_id, anchor: 'import-group-pane') + import_existing_group_path: new_group_path(parent_id: group.parent_id, anchor: 'import-group-pane'), + is_saas: Gitlab.com?.to_s } end diff --git a/app/models/integrations/hangouts_chat.rb b/app/models/integrations/hangouts_chat.rb index c903e8d9eb8..ad82f1b916f 100644 --- a/app/models/integrations/hangouts_chat.rb +++ b/app/models/integrations/hangouts_chat.rb @@ -58,9 +58,9 @@ module Integrations when Integrations::ChatMessage::NoteMessage message.target when Integrations::ChatMessage::IssueMessage - "issue #{Issue.reference_prefix}#{message.issue_iid}" + "issue #{message.project_name}#{Issue.reference_prefix}#{message.issue_iid}" when Integrations::ChatMessage::MergeMessage - "merge request #{MergeRequest.reference_prefix}#{message.merge_request_iid}" + "merge request #{message.project_name}#{MergeRequest.reference_prefix}#{message.merge_request_iid}" when Integrations::ChatMessage::PushMessage "push #{message.project_name}_#{message.ref}" when Integrations::ChatMessage::PipelineMessage diff --git a/data/deprecations/15-6-deprecate-post-api-v4-runner.yml b/data/deprecations/15-6-deprecate-post-api-v4-runner.yml index 8c2931faf78..17d07c0f119 100644 --- a/data/deprecations/15-6-deprecate-post-api-v4-runner.yml +++ b/data/deprecations/15-6-deprecate-post-api-v4-runner.yml @@ -10,7 +10,9 @@ The support for registration tokens and certain runner configuration arguments in the `POST` method operation on the `/api/v4/runners` endpoint is deprecated. This endpoint [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner with a GitLab instance at the instance, group, or project level through the API. Registration tokens, and support for certain configuration arguments, - will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: + will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). + + The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` diff --git a/data/deprecations/15-6-deprecate-runner-register-command.yml b/data/deprecations/15-6-deprecate-runner-register-command.yml index cb9b9f517cd..81e0f46078f 100644 --- a/data/deprecations/15-6-deprecate-runner-register-command.yml +++ b/data/deprecations/15-6-deprecate-runner-register-command.yml @@ -9,7 +9,8 @@ body: | # (required) Do not modify this line, instead modify the lines below. Registration tokens and certain configuration arguments in the command `gitlab-runner register` that [registers](https://docs.gitlab.com/runner/register/) a runner, are deprecated. Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, - will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: + will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). + The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` diff --git a/data/deprecations/15-6-deprecate-runner-register-token-k8s-operator.yml b/data/deprecations/15-6-deprecate-runner-register-token-k8s-operator.yml index da184cfbe43..61b117feb35 100644 --- a/data/deprecations/15-6-deprecate-runner-register-token-k8s-operator.yml +++ b/data/deprecations/15-6-deprecate-runner-register-token-k8s-operator.yml @@ -8,7 +8,8 @@ issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/382077 # (required) Link to the deprecation issue in GitLab body: | # (required) Do not modify this line, instead modify the lines below. The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and Kubernetes Vanilla Operator to install a runner on Kubernetes is deprecated. Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, - will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: + will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). + The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` diff --git a/db/docs/batched_background_migrations/mark_duplicate_npm_packages_for_destruction.yml b/db/docs/batched_background_migrations/mark_duplicate_npm_packages_for_destruction.yml new file mode 100644 index 00000000000..bd059876a25 --- /dev/null +++ b/db/docs/batched_background_migrations/mark_duplicate_npm_packages_for_destruction.yml @@ -0,0 +1,6 @@ +--- +migration_job_name: MarkDuplicateNpmPackagesForDestruction +description: It seeks duplicate npm packages and marks them for destruction. +feature_category: package_registry +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114695 +milestone: 16.1 diff --git a/db/docs/p_ci_builds.yml b/db/docs/p_ci_builds.yml index 2ad98d93485..ca01e89de09 100644 --- a/db/docs/p_ci_builds.yml +++ b/db/docs/p_ci_builds.yml @@ -9,6 +9,6 @@ classes: feature_categories: - continuous_integration description: Routing table for ci_builds -introduced_by_url: +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120873 milestone: '16.1' gitlab_schema: gitlab_ci diff --git a/db/post_migrate/20230524120241_add_temp_index_to_packages_on_project_id_when_npm_and_not_pending_destruction.rb b/db/post_migrate/20230524120241_add_temp_index_to_packages_on_project_id_when_npm_and_not_pending_destruction.rb new file mode 100644 index 00000000000..58a3a26d2eb --- /dev/null +++ b/db/post_migrate/20230524120241_add_temp_index_to_packages_on_project_id_when_npm_and_not_pending_destruction.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class AddTempIndexToPackagesOnProjectIdWhenNpmAndNotPendingDestruction < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + INDEX_NAME = 'tmp_idx_packages_on_project_id_when_npm_not_pending_destruction' + NPM_PACKAGE_TYPE = 2 + PENDING_DESTRUCTION_STATUS = 4 + + def up + # Temporary index to be removed in 16.2 https://gitlab.com/gitlab-org/gitlab/-/issues/414216 + add_concurrent_index( + :packages_packages, + :project_id, + name: INDEX_NAME, + where: "package_type = #{NPM_PACKAGE_TYPE} AND status <> #{PENDING_DESTRUCTION_STATUS}" + ) + end + + def down + remove_concurrent_index_by_name :packages_packages, INDEX_NAME + end +end diff --git a/db/post_migrate/20230524201454_queue_mark_duplicate_npm_packages_for_destruction.rb b/db/post_migrate/20230524201454_queue_mark_duplicate_npm_packages_for_destruction.rb new file mode 100644 index 00000000000..7460d93fd49 --- /dev/null +++ b/db/post_migrate/20230524201454_queue_mark_duplicate_npm_packages_for_destruction.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class QueueMarkDuplicateNpmPackagesForDestruction < Gitlab::Database::Migration[2.1] + MIGRATION = 'MarkDuplicateNpmPackagesForDestruction' + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 5000 + BATCH_CLASS_NAME = 'LooseIndexScanBatchingStrategy' + SUB_BATCH_SIZE = 500 + + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :packages_packages, + :project_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + batch_class_name: BATCH_CLASS_NAME, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :packages_packages, :project_id, []) + end +end diff --git a/db/schema_migrations/20230524120241 b/db/schema_migrations/20230524120241 new file mode 100644 index 00000000000..ae2b35d899f --- /dev/null +++ b/db/schema_migrations/20230524120241 @@ -0,0 +1 @@ +3709d98612eca5bfe996c20770b313322cde010fbf5b1e69c8ef688a456e1ace
\ No newline at end of file diff --git a/db/schema_migrations/20230524201454 b/db/schema_migrations/20230524201454 new file mode 100644 index 00000000000..1f4d660033c --- /dev/null +++ b/db/schema_migrations/20230524201454 @@ -0,0 +1 @@ +65b99cce227ef0a66cc83624170c680cc6edf72bd7e30e7c562104f4dbebabf9
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 195b75ec8ac..cd092431559 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -33563,6 +33563,8 @@ CREATE INDEX tmp_idx_for_feedback_comment_processing ON vulnerability_feedback U CREATE INDEX tmp_idx_for_vulnerability_feedback_migration ON vulnerability_feedback USING btree (id) WHERE ((migrated_to_state_transition = false) AND (feedback_type = 0)); +CREATE INDEX tmp_idx_packages_on_project_id_when_npm_not_pending_destruction ON packages_packages USING btree (project_id) WHERE ((package_type = 2) AND (status <> 4)); + CREATE INDEX tmp_idx_vulnerability_occurrences_on_id_where_report_type_7_99 ON vulnerability_occurrences USING btree (id) WHERE (report_type = ANY (ARRAY[7, 99])); CREATE INDEX tmp_index_ci_job_artifacts_on_expire_at_where_locked_unknown ON ci_job_artifacts USING btree (expire_at, job_id) WHERE ((locked = 2) AND (expire_at IS NOT NULL)); diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 88c9a669441..620bf1fa8c4 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -197,7 +197,7 @@ by the client. You are redirected to GitLab and signed in. ## Example configurations The following configurations illustrate how to set up OpenID with -different providers when using the GitLab Linux package installation. +different providers when using the Linux package installation. ### Configure Google diff --git a/doc/administration/consul.md b/doc/administration/consul.md index 3257c197445..c5b1cc0ed2a 100644 --- a/doc/administration/consul.md +++ b/doc/administration/consul.md @@ -76,7 +76,7 @@ To upgrade your Consul nodes, upgrade the GitLab package. Nodes should be: -- Members of a healthy cluster prior to upgrading the GitLab Linux package. +- Members of a healthy cluster prior to upgrading the Linux package. - Upgraded one node at a time. Identify any existing health issues in the cluster by running the following command diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 24a91a7a9c5..4757c730c7b 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -681,7 +681,8 @@ Data that was created on the primary while the secondary was paused is lost. If you are running GitLab 14.5 and later: -1. For each node outside of the **secondary** Kubernetes cluster using Omnibus such as PostgreSQL or Gitaly, SSH into the node and run one of the following commands: +1. For each node (such as PostgreSQL or Gitaly) outside of the **secondary** Kubernetes cluster using the Linux + package, SSH into the node and run one of the following commands: - To promote the **secondary** site node external to the Kubernetes cluster to primary: diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 501227f254a..22637ad84bc 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -13,11 +13,11 @@ This runbook is an [Experiment](../../../../policy/experiment-beta-support.md#ex ## Geo planned failover for a multi-node configuration -| Component | Configuration | -|-------------|-----------------| -| PostgreSQL | Omnibus-managed | -| Geo site | Multi-node | -| Secondaries | One | +| Component | Configuration | +|:------------|:-----------------------------| +| PostgreSQL | Managed by the Linux package | +| Geo site | Multi-node | +| Secondaries | One | This runbook guides you through a planned failover of a multi-node Geo site with one secondary. The following [2000 user reference architecture](../../../../administration/reference_architectures/2k_users.md) is assumed: @@ -205,7 +205,7 @@ follow these steps to avoid unnecessary data loss: NOTE: (**CentOS only**) In CentOS 6 or older, it is challenging to prevent GitLab from being - started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). + started if the machine reboots isn't available (see [issue 3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. NOTE: diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index 18d5aa4d130..ecece40f85a 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -13,11 +13,11 @@ This runbook is an [Experiment](../../../../policy/experiment-beta-support.md#ex ## Geo planned failover for a single-node configuration -| Component | Configuration | -|-------------|-----------------| -| PostgreSQL | Omnibus-managed | -| Geo site | Single-node | -| Secondaries | One | +| Component | Configuration | +|:------------|:-----------------------------| +| PostgreSQL | Managed by the Linux package | +| Geo site | Single-node | +| Secondaries | One | This runbook guides you through a planned failover of a single-node Geo site with one secondary. The following general architecture is assumed: @@ -190,7 +190,7 @@ follow these steps to avoid unnecessary data loss: NOTE: (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being - started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). + started if the machine reboots isn't available (see [issue 3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. NOTE: diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index da00ddc5cfb..e29ebb1bb64 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -246,10 +246,11 @@ secondary. If the site is paused, be sure to resume before promoting. This issue has been fixed in GitLab 13.4 and later. WARNING: -Pausing and resuming of replication is only supported for Geo installations using an -Omnibus GitLab-managed database. External databases are not supported. +Pausing and resuming of replication is only supported for Geo installations using a +Linux package-managed database. External databases are not supported. -In some circumstances, like during [upgrades](replication/upgrading_the_geo_sites.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. +In some circumstances, like during [upgrades](replication/upgrading_the_geo_sites.md) or a +[planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. Pausing and resuming replication is done via a command line tool from the node in the secondary site where the `postgresql` service is enabled. @@ -331,7 +332,7 @@ For answers to common questions, see the [Geo FAQ](replication/faq.md). ## Log files -Geo stores structured log messages in a `geo.log` file. For Omnibus GitLab +Geo stores structured log messages in a `geo.log` file. For Linux package installations, this file is at `/var/log/gitlab/gitlab-rails/geo.log`. This file contains information about when Geo attempts to sync repositories and files. Each line in the file contains a separate JSON entry that can be ingested into. For example, Elasticsearch or Splunk. diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 5fa6df393b9..5aee9bc5b83 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -12,7 +12,7 @@ type: howto NOTE: This is the final step in setting up a **secondary** Geo site. Stages of the setup process must be completed in the documented order. -If not, [complete all prior stages](../setup/index.md#using-omnibus-gitlab) before proceeding. +If not, [complete all prior stages](../setup/index.md#using-linux-package-installations) before proceeding. Make sure you [set up the database replication](../setup/database.md), and [configured fast lookup of authorized SSH keys](../../operations/fast_ssh_key_lookup.md) in **both primary and secondary sites**. diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md index c42130a62a7..bb47b8a3524 100644 --- a/doc/administration/geo/replication/disable_geo.md +++ b/doc/administration/geo/replication/disable_geo.md @@ -7,7 +7,7 @@ type: howto # Disabling Geo **(PREMIUM SELF)** -If you want to revert to a regular Omnibus setup after a test, or you have encountered a Disaster Recovery +If you want to revert to a regular Linux package installation setup after a test, or you have encountered a Disaster Recovery situation and you want to disable Geo momentarily, you can use these instructions to disable your Geo setup. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index 97c10b3ec0a..7897635e78f 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -143,7 +143,7 @@ The following are PostgreSQL upgrade validation tests we performed. we tested fresh installations of GitLab 13.3 with PostgreSQL 12 enabled and Geo installed. - Outcome: Setting up a Geo secondary required manual intervention because the `recovery.conf` file is no longer supported in PostgreSQL 12. We do not recommend deploying Geo with PostgreSQL 12 until - the appropriate changes have been made to Omnibus and verified. + the appropriate changes have been made to the Linux package and verified. - Follow up issues: - [Update `replicate-geo-database` to support PostgreSQL 12](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5575) - [Remove PostgreSQL 12 check in `replicate-geo-database` for 14.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5576) diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index ca4f1c36e69..54213139fb4 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -35,7 +35,7 @@ The **primary** and **secondary** Geo sites must be able to communicate to each Because of the additional complexity involved in setting up this configuration for PostgreSQL and Redis, it is not covered by this Geo multi-node documentation. -For more information on setting up a multi-node PostgreSQL cluster and Redis cluster using the Omnibus GitLab package, see: +For more information on setting up a multi-node PostgreSQL cluster and Redis cluster using the Linux package, see: - [Geo multi-node database replication](../setup/database.md#multi-node-database-replication) - [Redis multi-node documentation](../../redis/replication_and_failover.md) @@ -260,7 +260,7 @@ then make the following modifications: ``` NOTE: -If you had set up PostgreSQL cluster using the omnibus package and had set +If you had set up PostgreSQL cluster using the Linux package and had set `postgresql['sql_user_password'] = 'md5 digest of secret'`, keep in mind that `gitlab_rails['db_password']` and `geo_secondary['db_password']` contains the plaintext passwords. This is used to let the Rails diff --git a/doc/administration/geo/replication/security_review.md b/doc/administration/geo/replication/security_review.md index 92eff2faf60..e099c2b2e9d 100644 --- a/doc/administration/geo/replication/security_review.md +++ b/doc/administration/geo/replication/security_review.md @@ -127,9 +127,9 @@ from [owasp.org](https://owasp.org/). ### What details regarding required OS components and lock‐down needs have been defined? -- The supported installation method (Omnibus) packages most components itself. +- The supported Linux package installation method packages most components itself. - There are significant dependencies on the system-installed OpenSSH daemon (Geo - requires users to set up custom authentication methods) and the omnibus or + requires users to set up custom authentication methods) and the Linux package-provided or system-provided PostgreSQL daemon (it must be configured to listen on TCP, additional users and replication slots must be added, etc). - The process for dealing with security updates (for example, if there is a @@ -237,7 +237,7 @@ from [owasp.org](https://owasp.org/). - In transit, data should be encrypted, although the application does permit communication to proceed unencrypted. The two main transits are the **secondary** site's replication process for PostgreSQL, and for Git repositories/files. Both should - be protected using TLS, with the keys for that managed via Omnibus per existing + be protected using TLS, with the keys for that managed by the Linux package per existing configuration for end-user access to GitLab. ### What capabilities exist to detect the leakage of sensitive data? diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 7244222d2fc..1766b1bef8b 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -210,8 +210,8 @@ Geo finds the current Puma or Sidekiq node's Geo [site](../glossary.md) name in 1. Get the "Geo node name" (there is [an issue to rename the settings to "Geo site name"](https://gitlab.com/gitlab-org/gitlab/-/issues/335944)): - - Omnibus GitLab: Get the `gitlab_rails['geo_node_name']` setting. - - GitLab Helm Charts: Get the `global.geo.nodeName` setting (see [Charts with GitLab Geo](https://docs.gitlab.com/charts/advanced/geo/index.html)). + - Linux package: get the `gitlab_rails['geo_node_name']` setting. + - GitLab Helm charts: get the `global.geo.nodeName` setting (see [Charts with GitLab Geo](https://docs.gitlab.com/charts/advanced/geo/index.html)). 1. If that is not defined, then get the `external_url` setting. This name is used to look up the Geo site with the same **Name** in the **Geo Sites** @@ -600,7 +600,7 @@ To help us resolve this problem, consider commenting on ### Message: `FATAL: could not connect to the primary server: server certificate for "PostgreSQL" does not match host name` -This happens because the PostgreSQL certificate that the Omnibus GitLab package automatically creates contains +This happens because the PostgreSQL certificate that the Linux package automatically creates contains the Common Name `PostgreSQL`, but the replication is connecting to a different host and GitLab attempts to use the `verify-full` SSL mode by default. @@ -1158,7 +1158,7 @@ get_ctl_options': invalid option: --force (OptionParser::InvalidOption) ``` This can happen with XFS or file systems that list files in lexical order, because the -load order of the Omnibus GitLab command files can be different than expected, and a global function would get redefined. +load order of the Linux package command files can be different than expected, and a global function would get redefined. More details can be found in [the related issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6076). The workaround is to manually run the preflight checks and promote the database, by running @@ -1208,7 +1208,7 @@ This section documents common error messages reported in the Admin Area on the w GitLab cannot find or doesn't have permission to access the `database_geo.yml` configuration file. -In an Omnibus GitLab installation, the file should be in `/var/opt/gitlab/gitlab-rails/etc`. +In a Linux package installation, the file should be in `/var/opt/gitlab/gitlab-rails/etc`. If it doesn't exist or inadvertent changes have been made to it, run `sudo gitlab-ctl reconfigure` to restore it to its correct state. If this path is mounted on a remote volume, ensure your volume configuration @@ -1241,7 +1241,7 @@ This error message indicates that the replica database in the **secondary** site To restore the database and resume replication, you can do one of the following: - [Reset the Geo secondary site replication](#resetting-geo-secondary-site-replication). -- [Set up a new secondary Geo Omnibus instance](../setup/index.md#using-omnibus-gitlab). +- [Set up a new Geo secondary using the Linux package](../setup/index.md#using-linux-package-installations). If you set up a new secondary from scratch, you must also [remove the old site from the Geo cluster](remove_geo_site.md#removing-secondary-geo-sites). @@ -1260,7 +1260,7 @@ Make sure you follow the [Geo database replication](../setup/database.md) instru ### Geo database version (...) does not match latest migration (...) -If you are using Omnibus GitLab installation, something might have failed during upgrade. You can: +If you are using the Linux package installation, something might have failed during upgrade. You can: - Run `sudo gitlab-ctl reconfigure`. - Manually trigger the database migration by running: `sudo gitlab-rake db:migrate:geo` as root on the **secondary** site. @@ -1361,7 +1361,9 @@ If you have installed GitLab using the Linux package (Omnibus) and have configur - `15.6.0`-`15.6.3` - `15.7.0`-`15.7.1` -This is due to [a bug introduced in the included version of cURL](https://github.com/curl/curl/issues/10122) shipped with Omnibus GitLab 15.4.6 and later. You are encouraged to upgrade to a later version where this has been [fixed](https://about.gitlab.com/releases/2023/01/09/security-release-gitlab-15-7-2-released/). +This is due to [a bug introduced in the included version of cURL](https://github.com/curl/curl/issues/10122) shipped with +the Linux package 15.4.6 and later. You should upgrade to a later version where this has been +[fixed](https://about.gitlab.com/releases/2023/01/09/security-release-gitlab-15-7-2-released/). The bug causes all wildcard domains (`.example.com`) to be ignored except for the last on in the `no_proxy` environment variable list. Therefore, if for any reason you cannot upgrade to a newer version, you can work around the issue by moving your wildcard domain to the end of the list: diff --git a/doc/administration/geo/replication/version_specific_upgrades.md b/doc/administration/geo/replication/version_specific_upgrades.md index f8e013a8776..c0215c4551c 100644 --- a/doc/administration/geo/replication/version_specific_upgrades.md +++ b/doc/administration/geo/replication/version_specific_upgrades.md @@ -253,16 +253,16 @@ upgrade to GitLab 13.4 or later. ## Upgrading to GitLab 13.0 Upgrading to GitLab 13.0 requires GitLab 12.10 to already be using PostgreSQL -version 11. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +version 11. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). ## Upgrading to GitLab 12.10 GitLab 12.10 doesn't attempt to upgrade the embedded PostgreSQL server when using Geo, because the PostgreSQL upgrade requires downtime for secondaries while reinitializing streaming replication. It must be upgraded manually. For -the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). ## Upgrading to GitLab 12.9 @@ -273,8 +273,8 @@ The issue is fixed in GitLab 12.9.4. Upgrade to GitLab 12.9.4 or later. By default, GitLab 12.9 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.12, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -287,8 +287,8 @@ sudo touch /etc/gitlab/disable-postgresql-upgrade By default, GitLab 12.8 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.12, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -307,8 +307,8 @@ shipped in 12.7.5. By default, GitLab 12.7 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -321,8 +321,8 @@ sudo touch /etc/gitlab/disable-postgresql-upgrade By default, GitLab 12.6 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -335,8 +335,8 @@ sudo touch /etc/gitlab/disable-postgresql-upgrade By default, GitLab 12.5 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -349,8 +349,8 @@ sudo touch /etc/gitlab/disable-postgresql-upgrade By default, GitLab 12.4 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while -reinitializing streaming replication. For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +reinitializing streaming replication. For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before upgrading: @@ -365,13 +365,13 @@ WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to GitLab 12.4 or later. By default, GitLab 12.3 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can -fail. For more information, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +fail. For more information, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). Additionally, if the PostgreSQL upgrade doesn't fail, a successful upgrade requires downtime for secondaries while reinitializing streaming replication. -For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). ## Upgrading to GitLab 12.2 @@ -379,13 +379,13 @@ WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to GitLab 12.4 or later. By default, GitLab 12.2 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can -fail. For more information, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +fail. For more information, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). Additionally, if the PostgreSQL upgrade doesn't fail, a successful upgrade requires downtime for secondaries while reinitializing streaming replication. -For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). GitLab 12.2 includes the following minor PostgreSQL upgrades: @@ -410,13 +410,13 @@ WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to GitLab 12.4 or later. By default, GitLab 12.1 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can -fail. For more information, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +fail. For more information, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). Additionally, if the PostgreSQL upgrade doesn't fail, a successful upgrade requires downtime for secondaries while reinitializing streaming replication. -For the recommended procedure, see the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +For the recommended procedure, see how +[to upgrade a Geo instance](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). ## Upgrading to GitLab 12.0 diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index 1ca86a6a55d..3081d1c2485 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -182,7 +182,8 @@ Secondary proxying is enabled by default in GitLab 15.1 on a secondary site even Additionally, the `gitlab-workhorse` service polls `/api/v4/geo/proxy` every 10 seconds. In GitLab 15.2 and later, it is only polled once, if Geo is not enabled. Prior to GitLab 15.2, you can stop this polling by disabling secondary proxying. -You can disable the secondary proxying on each Geo site, separately, by following these steps with Omnibus-based packages: +You can disable the secondary proxying on each Geo site separately by following these steps on a Linux package +installation: 1. SSH into each application node (serving user traffic directly) on your secondary Geo site and add the following environment variable: diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index cf0c40dbbc5..31d0fbdffe0 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -12,14 +12,13 @@ GitLab database to a secondary site's database. You may have to change some values, based on attributes including your database's setup and size. NOTE: -If your GitLab installation uses external (not managed by Omnibus GitLab) -PostgreSQL instances, the Omnibus roles cannot perform all necessary -configuration steps. In this case, use the [Geo with external PostgreSQL instances](external_database.md) -process instead. +If your GitLab installation uses external PostgreSQL instances (not managed by a Linux package installation), +the roles cannot perform all necessary configuration steps. In this case, use the +[Geo with external PostgreSQL instances](external_database.md) process instead. NOTE: The stages of the setup process must be completed in the documented order. -If not, [complete all prior stages](../setup/index.md#using-omnibus-gitlab) before proceeding. +If not, [complete all prior stages](../setup/index.md#using-linux-package-installations) before proceeding. Ensure the **secondary** site is running the same version of GitLab Enterprise Edition as the **primary** site. Confirm you have added a license for a [Premium or Ultimate subscription](https://about.gitlab.com/pricing/) to your **primary** site. @@ -51,10 +50,10 @@ recover. See below for more details. The following guide assumes that: -- You are using Omnibus and therefore you are using PostgreSQL 12 or later, +- You are using the Linux package (so are using PostgreSQL 12 or later), which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/12/app-pgbasebackup.html). - You have a **primary** site already set up (the GitLab server you are - replicating from), running Omnibus' PostgreSQL (or equivalent version), and + replicating from), running PostgreSQL (or equivalent version) managed by your Linux package installation, and you have a new **secondary** site set up with the same [versions of PostgreSQL](../index.md#requirements-for-running-geo), OS, and GitLab on all sites. @@ -140,7 +139,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o postgresql['sql_replication_password'] = '<md5_hash_of_your_password>' ``` - If you are using an external database not managed by Omnibus GitLab, you need + If you are using an external database not managed by your Linux package installation, you need to create the `gitlab_replicator` user and define a password for that user manually: ```sql @@ -205,7 +204,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o NOTE: If you need to use `0.0.0.0` or `*` as the `listen_address`, you also must add `127.0.0.1/32` to the `postgresql['md5_auth_cidr_addresses']` setting, to allow Rails to connect through - `127.0.0.1`. For more information, see [omnibus-5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). + `127.0.0.1`. For more information, see [issue 5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). Depending on your network configuration, the suggested addresses may be incorrect. If your **primary** site and **secondary** sites connect over a local @@ -374,7 +373,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o to the private key, which is **only** present on the **primary** site. 1. Test that the `gitlab-psql` user can connect to the **primary** site's database - (the default Omnibus database name is `gitlabhq_production`): + (the default database name is `gitlabhq_production` on a Linux package installation): ```shell sudo \ @@ -456,8 +455,8 @@ Below is a script that connects the database on the **secondary** site to the database on the **primary** site. This script replicates the database and creates the needed files for streaming replication. -The directories used are the defaults that are set up in Omnibus. If you have -changed any defaults, configure the script accordingly, replacing any directories and paths. +The directories used are the defaults that are set up in a Linux package installation. If you have +changed any defaults, configure the script accordingly (replacing any directories and paths). WARNING: Make sure to run this on the **secondary** site as it removes all PostgreSQL's @@ -537,12 +536,12 @@ You should use PgBouncer if you use GitLab in a highly available configuration with a cluster of nodes supporting a Geo **primary** site and two other clusters of nodes supporting a Geo **secondary** site. You need two PgBouncer nodes: one for the main database and the other for the tracking database. For more information, -see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). +see [the relevant documentation](../../postgresql/replication_and_failover.md). ### Changing the replication password To change the password for the [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication) -when using Omnibus-managed PostgreSQL instances: +when using PostgreSQL instances managed by a Linux package installation: On the GitLab Geo **primary** site: @@ -628,7 +627,7 @@ to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your ### Migrating a single PostgreSQL node to Patroni -Before the introduction of Patroni, Geo had no Omnibus support for HA setups on the **secondary** site. +Before the introduction of Patroni, Geo had no support for Linux package installations for HA setups on the **secondary** site. With Patroni, this support is now possible. To migrate the existing PostgreSQL to Patroni: @@ -639,7 +638,7 @@ With Patroni, this support is now possible. To migrate the existing PostgreSQL t 1. [Configure a Standby Cluster](#step-4-configure-a-standby-cluster-on-the-secondary-site) on that single node machine. -You end up with a “Standby Cluster” with a single node. That allows you to add additional Patroni nodes by following the same instructions above. +You end up with a _Standby Cluster_ with a single node. That allows you to add additional Patroni nodes by following the same instructions above. ### Patroni support @@ -649,7 +648,7 @@ Using Patroni on a **secondary** site is optional and you don't have to use the nodes on each Geo site. For instructions on how to set up Patroni on the primary site, see the -[PostgreSQL replication and failover with Omnibus GitLab](../../postgresql/replication_and_failover.md#patroni) page. +[relevant documentation](../../postgresql/replication_and_failover.md#patroni). #### Configuring Patroni cluster for a Geo secondary site @@ -743,8 +742,8 @@ Leader is elected on the primary site, you should set up a TCP internal load balancer. This load balancer provides a single endpoint for connecting to the Patroni cluster's Leader. -The Omnibus GitLab packages do not include a Load Balancer. Here's how you -could do it with [HAProxy](https://www.haproxy.org/). +Linux packages do not include a Load Balancer. Here's how you could do it with +[HAProxy](https://www.haproxy.org/). The following IPs and names are used as an example: @@ -787,7 +786,7 @@ three Consul nodes and a minimum of one PgBouncer node. However, it is recommend one PgBouncer node per database node. An internal load balancer (TCP) is required when there is more than one PgBouncer service node. The internal load balancer provides a single endpoint for connecting to the PgBouncer cluster. For more information, -see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). +see [the relevant documentation](../../postgresql/replication_and_failover.md). On each node running a PgBouncer instance on the **secondary** site: @@ -921,7 +920,7 @@ For each node running a Patroni instance on the secondary site: ### Migrating a single tracking database node to Patroni -Before the introduction of Patroni, Geo provided no Omnibus support for HA setups on +Before the introduction of Patroni, Geo provided no support for Linux package installations for HA setups on the secondary site. With Patroni, it's now possible to support HA setups. However, some restrictions in Patroni @@ -935,7 +934,7 @@ synchronization is required. **Secondary** sites use a separate PostgreSQL installation as a tracking database to keep track of replication status and automatically recover from potential replication issues. -Omnibus automatically configures a tracking database when `roles(['geo_secondary_role'])` is set. +The Linux package automatically configures a tracking database when `roles(['geo_secondary_role'])` is set. If you want to run this database in a highly available configuration, don't use the `geo_secondary_role` above. Instead, follow the instructions below. @@ -945,7 +944,7 @@ If you want to run the Geo tracking database on a single node, see [Configure th A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes: two Patroni nodes, and one PgBouncer node on the secondary site. -Because of [omnibus-6587](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6587), Consul can't track multiple +Because of [issue 6587](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6587), Consul can't track multiple services, so these must be different than the nodes used for the Standby Cluster database. Be sure to use [password credentials](../../postgresql/replication_and_failover.md#database-authorization-for-patroni) diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index 65ea2e6e5af..0ca1356246b 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -6,17 +6,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Geo with external PostgreSQL instances **(PREMIUM SELF)** -This document is relevant if you are using a PostgreSQL instance that is *not -managed by Omnibus*. This includes cloud-managed instances like Amazon RDS, or +This document is relevant if you are using a PostgreSQL instance that is not +managed by the Linux package. This includes cloud-managed instances like Amazon RDS, or manually installed and configured PostgreSQL instances. Ensure that you are using one of the PostgreSQL versions that -[Omnibus ships with](../../package_information/postgresql_versions.md) +the [Linux package ships with](../../package_information/postgresql_versions.md) to [avoid version mismatches](../index.md#requirements-for-running-geo) in case a Geo site has to be rebuilt. NOTE: -We strongly recommend running Omnibus-managed instances as they are actively +We strongly recommend running instances installed using the Linux package as they are actively developed and tested. We aim to be compatible with most external (not managed by Omnibus) databases but we do not guarantee compatibility. @@ -62,8 +62,8 @@ developed and tested. We aim to be compatible with most external To set up an external database, you can either: -- Set up [streaming replication](https://www.postgresql.org/docs/12/warm-standby.html#STREAMING-REPLICATION-SLOTS) yourself (for example Amazon RDS, or bare metal not managed by Omnibus). -- Perform the Omnibus configuration manually as follows. +- Set up [streaming replication](https://www.postgresql.org/docs/12/warm-standby.html#STREAMING-REPLICATION-SLOTS) yourself (for example Amazon RDS, or bare metal not managed by the Linux package). +- Manually perform the configuration of your Linux package installations as follows. #### Leverage your cloud provider's tools to replicate the primary database @@ -142,7 +142,7 @@ hot_standby = on ### Configure **secondary** site to use the external read-replica -With Omnibus, the +With Linux package installations, the [`geo_secondary_role`](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles) has three main functions: @@ -185,9 +185,9 @@ To configure the connection to the external read-replica database and enable Log **Secondary** sites use a separate PostgreSQL installation as a tracking database to keep track of replication status and automatically recover from -potential replication issues. Omnibus automatically configures a tracking database +potential replication issues. The Linux package automatically configures a tracking database when `roles ['geo_secondary_role']` is set. -If you want to run this database external to Omnibus GitLab, use the following instructions. +If you want to run this database external to your Linux package installation, use the following instructions. If you are using a cloud-managed service for the tracking database, you may need to grant additional roles to your tracking database user (by default, this is diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md index 94df58cc9eb..e401f07795f 100644 --- a/doc/administration/geo/setup/index.md +++ b/doc/administration/geo/setup/index.md @@ -21,9 +21,9 @@ type: howto - Confirm the **primary** and **secondary** site storage configurations match. If the primary Geo site uses object storage, the secondary Geo site must use it too. For more information, see [Geo with Object storage](../replication/object_storage.md). - Ensure clocks are synchronized between the **primary** site and the **secondary** site. Synchronized clocks are required for Geo to function correctly. For example, if the clock drift between the **primary** and **secondary** sites exceeds 1 minute, replication fails. -## Using Omnibus GitLab +## Using Linux package installations -If you installed GitLab using the Omnibus packages (highly recommended), the process for setting up Geo depends on whether you need to set up +If you installed GitLab using the Linux package (highly recommended), the process for setting up Geo depends on whether you need to set up a single-node Geo site or a multi-node Geo site. ### Single-node Geo sites diff --git a/doc/administration/get_started.md b/doc/administration/get_started.md index 26848f454b5..60291732a20 100644 --- a/doc/administration/get_started.md +++ b/doc/administration/get_started.md @@ -172,7 +172,7 @@ If your GitLab server contains a lot of Git repository data, you may find the Gi Slowness typically starts at a Git repository data size of around 200 GB. In this case, you might consider using file system snapshots as part of your backup strategy. For example, consider a GitLab server with the following components: -- Using the GitLab Linux package. +- Using the Linux package. - Hosted on AWS with an EBS drive containing an ext4 file system mounted at `/var/opt/gitlab`. The EC2 instance meets the requirements for an application data backup by taking an EBS snapshot. The backup includes all repositories, uploads, and PostgreSQL data. diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 2afa0fb233a..08aeb149454 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -167,7 +167,7 @@ These reference architectures were built and tested on Google Cloud Platform (GC [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) CPU platform as a lowest common denominator baseline ([Sysbench benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks)). -Newer, similarly-sized CPUs are supported and may have improved performance as a result. For Omnibus GitLab environments, +Newer, similarly-sized CPUs are supported and may have improved performance as a result. For Linux package environments, ARM-based equivalents are also supported. NOTE: @@ -240,7 +240,7 @@ for more information and guidance. that to achieve full High Availability, a third-party PostgreSQL database solution is required. We hope to offer a built-in solution for these restrictions in the future. In the meantime, a non-HA PostgreSQL server -can be set up using Omnibus GitLab as the specifications reflect. Refer to the following issues for more information: +can be set up using the Linux package as the specifications reflect. Refer to the following issues for more information: - [`omnibus-gitlab#7292`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). - [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398). @@ -325,8 +325,8 @@ Additionally, the following cloud provider services are validated and supported If you choose to use a third party external service: -1. Note that the HA Omnibus PostgreSQL setup encompasses PostgreSQL, PgBouncer and Consul. All of these components would no longer be required when using a third party external service. -1. The number of nodes required to achieve HA may differ depending on the service compared to Omnibus and doesn't need to match accordingly. +1. Note that the HA Linux package PostgreSQL setup encompasses PostgreSQL, PgBouncer and Consul. All of these components would no longer be required when using a third party external service. +1. The number of nodes required to achieve HA may differ depending on the service compared to the Linux package and doesn't need to match accordingly. 1. However, if [Database Load Balancing](../postgresql/database_load_balancing.md) via Read Replicas is desired for further improved performance it's recommended to follow the node count for the Reference Architecture. 1. If [GitLab Geo](../geo/index.md) is to be used the service will need to support Cross Region replication @@ -357,8 +357,8 @@ the harder it is to get support for it. With any deviation, you're introducing a layer of complexity that adds challenges to finding out where potential issues might lie. -The reference architectures use the official GitLab Linux packages (Omnibus -GitLab) or [Helm Charts](https://docs.gitlab.com/charts/) to install and configure the various components. The components are +The reference architectures use the official Linux packages or [Helm Charts](https://docs.gitlab.com/charts/) to +install and configure the various components. The components are installed on separate machines (virtualized or bare metal), with machine hardware requirements listed in the "Configuration" column and equivalent VM standard sizes listed in GCP/AWS/Azure columns of each [available reference architecture](#available-reference-architectures). @@ -418,9 +418,10 @@ Testing occurs against all reference architectures and cloud providers in an aut Network latency on the test environments between components on all Cloud Providers were measured at <5 ms. This is shared as an observation and not as an implicit recommendation. -We aim to have a "test smart" approach where architectures tested have a good range that can also apply to others. Testing focuses on 10k Omnibus on GCP as the testing has shown this is a good bellwether for the other architectures and cloud providers as well as Cloud Native Hybrids. +We aim to have a "test smart" approach where architectures tested have a good range that can also apply to others. Testing focuses on a 10k Linux package +installation on GCP as the testing has shown this is a good bellwether for the other architectures and cloud providers as well as Cloud Native Hybrids. -The Standard Reference Architectures are designed to be platform-agnostic, with everything being run on VMs via [Omnibus GitLab](https://docs.gitlab.com/omnibus/). While testing occurs primarily on GCP, ad-hoc testing has shown that they perform similarly on hardware with equivalent specs on other Cloud Providers or if run on premises (bare-metal). +The Standard Reference Architectures are designed to be platform-agnostic, with everything being run on VMs through [the Linux package](https://docs.gitlab.com/omnibus/). While testing occurs primarily on GCP, ad-hoc testing has shown that they perform similarly on hardware with equivalent specs on other Cloud Providers or if run on premises (bare-metal). Testing on these reference architectures is performed with the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) @@ -472,11 +473,11 @@ table.test-coverage th { <th style="text-align: center" colspan="2" scope="colgroup">Azure</th> </tr> <tr> - <th scope="col">Omnibus</th> + <th scope="col">Linux package</th> <th scope="col">Cloud Native Hybrid</th> - <th scope="col">Omnibus</th> + <th scope="col">Linux package</th> <th scope="col">Cloud Native Hybrid</th> - <th scope="col">Omnibus</th> + <th scope="col">Linux package</th> </tr> <tr> <th scope="row">1k</th> @@ -538,7 +539,7 @@ table.test-coverage th { ## Cost to run -As a starting point, the following table details initial costs for the different reference architectures across GCP, AWS, and Azure via Omnibus. +As a starting point, the following table details initial costs for the different reference architectures across GCP, AWS, and Azure through the Linux package. NOTE: Due to the nature of Cloud Native Hybrid, it's not possible to give a static cost calculation. @@ -555,9 +556,9 @@ Bare-metal costs are also not included here as it varies widely depending on eac <th style="text-align: center" scope="colgroup">Azure</th> </tr> <tr> - <th scope="col">Omnibus</th> - <th scope="col">Omnibus</th> - <th scope="col">Omnibus</th> + <th scope="col">Linux package</th> + <th scope="col">Linux package</th> + <th scope="col">Linux package</th> </tr> <tr> <th scope="row">1k</th> diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index 7b9145050bf..f6a6d105a17 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -30,6 +30,10 @@ To connect to an external repository: 1. Select **GitHub** or **Repository by URL**. 1. Complete the fields. +If the **Run CI/CD for external repository** option is not available, the GitLab instance +might not have any import sources configured. Ask an administrator for your instance to check +the [import sources configuration](../../user/admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources). + ## Pipelines for external pull requests > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/65139) in GitLab 12.3. diff --git a/doc/ci/runners/register_runner.md b/doc/ci/runners/register_runner.md index f53297099d7..bec80faecff 100644 --- a/doc/ci/runners/register_runner.md +++ b/doc/ci/runners/register_runner.md @@ -92,8 +92,10 @@ To generate an authentication token for a project runner: WARNING: The ability to pass a runner registration token, and support for certain configuration arguments was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/380872) in GitLab 15.6. Authentication tokens -should be used instead to register runners. Registration tokens, and support for certain configuration arguments -will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for `glrt-` tokens are `--locked`, `--access-level`, `--run-untagged`, `--maximum-timeout`, `--paused`, `--tag-list`, and `--maintenance-note`. This change is a breaking +should be used instead to register runners. Registration tokens, and support for certain configuration +arguments, will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](new_creation_workflow.md). +The configuration arguments disabled for `glrt-` tokens will be `--locked`, `--access-level`, +`--run-untagged`, `--maximum-timeout`, `--paused`, `--tag-list`, and `--maintenance-note`. This change is a breaking change. ### For a shared runner diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index c3de9d29dd1..eb26c0a73a4 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -20,8 +20,8 @@ For Free, Premium, and Ultimate plan customers, jobs on these instances consume | Runner Tag | vCPUs | Memory | Storage | |-----------------------------------------------|-------|--------|---------| | `saas-linux-small-amd64` | 2 | 8 GB | 25 GB | -| `saas-linux-medium-amd64` **(PREMIUM SAAS)** | 4 | 16 GB | 25 GB | -| `saas-linux-large-amd64` **(PREMIUM SAAS)** | 8 | 32 GB | 25 GB | +| `saas-linux-medium-amd64` **(PREMIUM SAAS)** | 4 | 16 GB | 50 GB | +| `saas-linux-large-amd64` **(PREMIUM SAAS)** | 8 | 32 GB | 100 GB | The `small` machine type is set as default. If no [tag](../../yaml/index.md#tags) keyword in your `.gitlab-ci.yml` file is specified, the jobs will run on this default runner. diff --git a/doc/development/service_ping/troubleshooting.md b/doc/development/service_ping/troubleshooting.md index 74e826a2208..57c4408ca84 100644 --- a/doc/development/service_ping/troubleshooting.md +++ b/doc/development/service_ping/troubleshooting.md @@ -71,7 +71,7 @@ checking the configuration file of your GitLab instance: settings. The configuration file in which Service Ping can be disabled depends on your installation and deployment method, but is typically one of the following: - - `/etc/gitlab/gitlab.rb` for Omnibus GitLab Linux Package and Docker. + - `/etc/gitlab/gitlab.rb` for Linux package installations and Docker. - `charts.yaml` for GitLab Helm and cloud-native Kubernetes deployments. - `gitlab.yml` for GitLab installations from source. diff --git a/doc/install/aws/manual_install_aws.md b/doc/install/aws/manual_install_aws.md index bd81e0583b5..92375fff59e 100644 --- a/doc/install/aws/manual_install_aws.md +++ b/doc/install/aws/manual_install_aws.md @@ -8,23 +8,23 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Installing a GitLab POC on Amazon Web Services (AWS) **(FREE SELF)** -This page offers a walkthrough of a common configuration for GitLab on AWS using the official GitLab Linux package. You should customize it to accommodate your needs. +This page offers a walkthrough of a common configuration for GitLab on AWS using the official Linux package. You should customize it to accommodate your needs. NOTE: -For organizations with 1,000 users or less, the recommended AWS installation method is to launch an EC2 single box [Omnibus Installation](https://about.gitlab.com/install/) and implement a snapshot strategy for backing up the data. See the [1,000 user reference architecture](../../administration/reference_architectures/1k_users.md) for more information. +For organizations with 1,000 users or less, the recommended AWS installation method is to launch an EC2 single box [Linux package installation](https://about.gitlab.com/install/) and implement a snapshot strategy for backing up the data. See the [1,000 user reference architecture](../../administration/reference_architectures/1k_users.md) for more information. ## Getting started for production-grade GitLab NOTE: This document is an installation guide for a proof of concept instance. It is not a reference architecture and it does not result in a highly available configuration. -Following this guide exactly results in a proof of concept instance that roughly equates to a **scaled down** version of a **two availability zone implementation** of the **Non-HA** [Omnibus 2000 User Reference Architecture](../../administration/reference_architectures/2k_users.md). The 2K reference architecture is not HA because it is primarily intended to provide some scaling while keeping costs and complexity low. The [3000 User Reference Architecture](../../administration/reference_architectures/3k_users.md) is the smallest size that is GitLab HA. It has additional service roles to achieve HA, most notably it uses Gitaly Cluster to achieve HA for Git repository storage and specifies triple redundancy. +Following this guide exactly results in a proof of concept instance that roughly equates to a **scaled down** version of a **two availability zone implementation** of the **Non-HA** [2000 User Reference Architecture](../../administration/reference_architectures/2k_users.md). The 2K reference architecture is not HA because it is primarily intended to provide some scaling while keeping costs and complexity low. The [3000 User Reference Architecture](../../administration/reference_architectures/3k_users.md) is the smallest size that is GitLab HA. It has additional service roles to achieve HA, most notably it uses Gitaly Cluster to achieve HA for Git repository storage and specifies triple redundancy. -GitLab maintains and tests two main types of Reference Architectures. The **Omnibus architectures** are implemented on instance compute while **Cloud Native Hybrid architectures** maximize the use of a Kubernetes cluster. Cloud Native Hybrid reference architecture specifications are addendum sections to the Reference Architecture size pages that start by describing the Omnibus architecture. For example, the 3000 User Cloud Native Reference Architecture is in the subsection titled [Cloud Native Hybrid reference architecture with Helm Charts (alternative)](../../administration/reference_architectures/3k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) in the 3000 User Reference Architecture page. +GitLab maintains and tests two main types of Reference Architectures. The **Linux package architectures** are implemented on instance compute while **Cloud Native Hybrid architectures** maximize the use of a Kubernetes cluster. Cloud Native Hybrid reference architecture specifications are addendum sections to the Reference Architecture size pages that start by describing the Linux package architecture. For example, the 3000 User Cloud Native Reference Architecture is in the subsection titled [Cloud Native Hybrid reference architecture with Helm Charts (alternative)](../../administration/reference_architectures/3k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) in the 3000 User Reference Architecture page. -### Getting started for production-grade Omnibus GitLab +### Getting started for production-grade Linux package installations -The Infrastructure as Code tooling [GitLab Environment Tool (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/tree/main) is the best place to start for building Omnibus GitLab on AWS and most especially if you are targeting an HA setup. While it does not automate everything, it does complete complex setups like Gitaly Cluster for you. GET is open source so anyone can build on top of it and contribute improvements to it. +The Infrastructure as Code tooling [GitLab Environment Tool (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/tree/main) is the best place to start for building using the Linux package on AWS and most especially if you are targeting an HA setup. While it does not automate everything, it does complete complex setups like Gitaly Cluster for you. GET is open source so anyone can build on top of it and contribute improvements to it. ### Getting started for production-grade Cloud Native Hybrid GitLab @@ -32,7 +32,7 @@ For the Cloud Native Hybrid architectures there are two Infrastructure as Code o ## Introduction -For the most part, we make use of Omnibus GitLab in our setup, but we also leverage native AWS services. Instead of using the Omnibus bundled PostgreSQL and Redis, we use Amazon RDS and ElastiCache. +For the most part, we make use of the Linux package in our setup, but we also leverage native AWS services. Instead of using the Linux package-bundled PostgreSQL and Redis, we use Amazon RDS and ElastiCache. In this guide, we go through a multi-node setup where we start by configuring our Virtual Private Cloud and subnets to later integrate @@ -783,7 +783,7 @@ For GitLab 12.1 and earlier, use `gitlab-rake gitlab:backup:create`. To restore GitLab, first review the [restore documentation](../../raketasks/backup_restore.md#restore-gitlab), and primarily the restore prerequisites. Then, follow the steps under the -[Omnibus installations section](../../raketasks/restore_gitlab.md#restore-for-omnibus-gitlab-installations). +[Linux package installations section](../../raketasks/restore_gitlab.md#restore-for-omnibus-gitlab-installations). ## Updating GitLab @@ -831,7 +831,7 @@ to request additional material: GitLab supports several different types of clustering. - [Geo replication](../../administration/geo/index.md): Geo is the solution for widely distributed development teams. -- [Omnibus GitLab](https://docs.gitlab.com/omnibus/) - Everything you must know +- [Linux package](https://docs.gitlab.com/omnibus/) - Everything you must know about administering your GitLab instance. - [Add a license](../../user/admin_area/license.md): Activate all GitLab Enterprise Edition functionality with a license. diff --git a/doc/install/docker.md b/doc/install/docker.md index 0d5b0e5d7b0..ab1aec98b16 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -783,7 +783,7 @@ can't create Thread: Operation not permitted This error occurs when running a container built with newer `glibc` versions on a [host that doesn't have support for the new clone3 function](https://github.com/moby/moby/issues/42680). In GitLab 16.0 and later, the container image includes -the Ubuntu 22.04 GitLab Linux package which is built with this newer `glibc`. +the Ubuntu 22.04 Linux package which is built with this newer `glibc`. This problem is fixed with newer container runtime tools like [Docker 20.10.10](https://github.com/moby/moby/pull/42836). diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index e492b5d75ce..7b90d4ad19c 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -7,7 +7,7 @@ description: 'Learn how to install a GitLab instance on Google Cloud Platform.' # Installing GitLab on Google Cloud Platform **(FREE SELF)** -You can install GitLab on a [Google Cloud Platform (GCP)](https://cloud.google.com/) using the official GitLab Linux package. You should customize it to accommodate your needs. +You can install GitLab on a [Google Cloud Platform (GCP)](https://cloud.google.com/) using the official Linux package. You should customize it to accommodate your needs. NOTE: To deploy production-ready GitLab on @@ -92,7 +92,7 @@ here's how you configure GitLab to be aware of the change: In the future you might want to set up [connecting with an SSH key](https://cloud.google.com/compute/docs/instances/connecting-to-instance) instead. -1. Edit the configuration file of Omnibus GitLab using your favorite text editor: +1. Edit the configuration file of the Linux package using your favorite text editor: ```shell sudo vim /etc/gitlab/gitlab.rb @@ -123,14 +123,14 @@ Although not needed, it's strongly recommended to secure GitLab with a ### Configuring the email SMTP settings You must configure the email SMTP settings correctly otherwise GitLab cannot send notification emails, like comments, and password changes. -Check the [Omnibus documentation](https://docs.gitlab.com/omnibus/settings/smtp.html#smtp-settings) how to do so. +Check the [Linux package documentation](https://docs.gitlab.com/omnibus/settings/smtp.html#smtp-settings) how to do so. ## Further reading GitLab can be configured to authenticate with other OAuth providers, like LDAP, SAML, and Kerberos. Here are some documents you might be interested in reading: -- [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/) +- [Linux package documentation](https://docs.gitlab.com/omnibus/) - [Integration documentation](../../integration/index.md) - [GitLab Pages configuration](../../administration/pages/index.md) - [GitLab Container Registry configuration](../../administration/packages/container_registry.md) diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 4a7c96d1330..64dfd3e6044 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -155,10 +155,10 @@ of GitLab Support or other GitLab engineers. ## Puma settings The recommended settings for Puma are determined by the infrastructure on which it's running. -The GitLab Linux package defaults to the recommended Puma settings. Regardless of installation method, you can +The Linux package defaults to the recommended Puma settings. Regardless of installation method, you can tune the Puma settings: -- If you're using the GitLab Linux package, see [Puma settings](../administration/operations/puma.md) +- If you're using the Linux package, see [Puma settings](../administration/operations/puma.md) for instructions on changing the Puma settings. - If you're using the GitLab Helm chart, see the [`webservice` chart](https://docs.gitlab.com/charts/charts/gitlab/webservice/index.html). diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index e1183f62225..0f9192f9a84 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -10,7 +10,7 @@ NOTE: This document applies to GitLab 11.0 and later. You can run a [GitLab Mattermost](https://gitlab.com/gitlab-org/gitlab-mattermost) -service on your GitLab server. Mattermost is not part of the single application that GitLab is. There is a good integration between [Mattermost and GitLab](https://mattermost.com/solutions/mattermost-gitlab/), and our Omnibus installer allows you to easily install it. But it is a separate application from a separate company. +service on your GitLab server. Mattermost is not part of the single application that GitLab is. There is a good integration between [Mattermost and GitLab](https://mattermost.com/solutions/mattermost-gitlab/), and our Linux package allows you to easily install it. But it is a separate application from a separate company. ## Prerequisites @@ -38,7 +38,7 @@ GitLab Mattermost is disabled by default. To enable it: 1. Confirm that GitLab Mattermost is reachable at `https://mattermost.example.com` and authorized to connect to GitLab. Authorizing Mattermost with GitLab allows users to use GitLab as an SSO provider. -The Omnibus GitLab package attempts to automatically authorize GitLab Mattermost with GitLab if the applications are running on the same server. +The Linux package attempts to automatically authorize GitLab Mattermost with GitLab if the applications are running on the same server. Automatic authorization requires access to the GitLab database. If the GitLab database is not available you need to manually authorize GitLab Mattermost for access to GitLab using the process described in the [Authorize GitLab Mattermost section](#authorize-gitlab-mattermost). @@ -86,7 +86,7 @@ Once the configuration is set, run `sudo gitlab-ctl reconfigure` to apply the ch ## Running GitLab Mattermost on its own server If you want to run GitLab and GitLab Mattermost on two separate servers the GitLab services are still set up on your GitLab Mattermost server, but they do not accept user requests or -consume system resources. You can use the following settings and configuration details on the GitLab Mattermost server to effectively disable the GitLab service bundled into the Omnibus package. +consume system resources. You can use the following settings and configuration details on the GitLab Mattermost server to effectively disable the GitLab service bundled into the Linux package. ```ruby mattermost_external_url 'http://mattermost.example.com' @@ -142,7 +142,7 @@ Save the changes and then run `sudo gitlab-ctl reconfigure`. If there are no err ## Specify numeric user and group identifiers -Omnibus GitLab creates a user and group `mattermost`. You can specify the +The Linux pacakage creates a user and group `mattermost`. You can specify the numeric identifiers for these users in `/etc/gitlab/gitlab.rb` as follows: ```ruby @@ -167,7 +167,7 @@ Run `sudo gitlab-ctl reconfigure` to apply the changes. ## Connecting to the bundled PostgreSQL database -If you need to connect to the bundled PostgreSQL database and are using the default Omnibus GitLab database configuration, you can connect as +If you need to connect to the bundled PostgreSQL database and are using the default Linux package database configuration, you can connect as the PostgreSQL superuser: ```shell @@ -176,14 +176,14 @@ sudo gitlab-psql -d mattermost_production ## Back up GitLab Mattermost -GitLab Mattermost is not included in the regular [Omnibus GitLab backup](../../raketasks/backup_restore.md) Rake task. +GitLab Mattermost is not included in the regular [Linux package backup](../../raketasks/backup_restore.md) Rake task. The general Mattermost [backup and disaster recovery](https://docs.mattermost.com/deploy/backup-disaster-recovery.html) documentation can be used as a guide on what needs to be backed up. ### Back up the bundled PostgreSQL database -If you need to back up the bundled PostgreSQL database and are using the default Omnibus GitLab database configuration, you can back up using this command: +If you need to back up the bundled PostgreSQL database and are using the default Linux package database configuration, you can back up using this command: ```shell sudo -i -u gitlab-psql -- /opt/gitlab/embedded/bin/pg_dump -h /var/opt/gitlab/postgresql mattermost_production | gzip > mattermost_dbdump_$(date --rfc-3339=date).sql.gz @@ -286,8 +286,6 @@ Password: ## Configuring GitLab and Mattermost integrations -As of 12.3, the Mattermost GitLab plugin is shipped with Omnibus GitLab: [Mattermost Plugin for GitLab documentation](https://github.com/mattermost/mattermost-plugin-gitlab). - You can use the plugin to subscribe Mattermost to receive notifications about issues, merge requests, and pull requests as well as personal notifications regarding merge request reviews, unread messages, and task assignments. If you want to use slash commands to perform actions such as creating and viewing issues, or to trigger deployments use GitLab [Mattermost slash commands](../../user/project/integrations/mattermost_slash_commands.md). @@ -359,22 +357,22 @@ When upgrading the Mattermost version, it is essential to check the [Important Upgrade Notes](https://docs.mattermost.com/administration/important-upgrade-notes.html) for Mattermost to address any changes or migrations that need to be performed. -Starting with GitLab 11.0, GitLab Mattermost can be upgraded through the regular Omnibus GitLab update process. When upgrading previous versions of +Starting with GitLab 11.0, GitLab Mattermost can be upgraded through the regular Linux package update process. When upgrading previous versions of GitLab, the update process can only be used if Mattermost configuration settings have not been changed outside of GitLab. That is, no changes to the Mattermost `config.json` file have been made - either directly or via the Mattermost **System Console**, which saves changes to `config.json`. -If you are upgrading to at least GitLab 11.0 or have only configured Mattermost using `gitlab.rb`, you can upgrade GitLab using Omnibus and then run `gitlab-ctl reconfigure` to upgrade GitLab Mattermost to the latest version. +If you are upgrading to at least GitLab 11.0 or have only configured Mattermost using `gitlab.rb`, you can upgrade GitLab using the Linux package and then run `gitlab-ctl reconfigure` to upgrade GitLab Mattermost to the latest version. If this is not the case, there are two options: 1. Update [`gitlab.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template#L706) with the changes done to `config.json`. This might require adding some parameters as not all - settings in `config.json` are available in `gitlab.rb`. Once complete, Omnibus GitLab should be + settings in `config.json` are available in `gitlab.rb`. Once complete, the Linux package should be able to upgrade GitLab Mattermost from one version to the next. -1. Migrate Mattermost outside of the directory controlled by Omnibus GitLab so it can be administered +1. Migrate Mattermost outside of the directory controlled by the Linux package so it can be administered and upgraded independently. Follow the [Mattermost Migration Guide](https://docs.mattermost.com/administration/migrating.html) to move your Mattermost configuration settings and data to another directory or server independent - from Omnibus GitLab. + from the Linux package. For a complete list of upgrade notices and special considerations for older versions, see the [Mattermost documentation](https://docs.mattermost.com/administration/important-upgrade-notes.html). @@ -387,7 +385,7 @@ GitLab 15.10 ships with Mattermost 7.8. Before upgrading, [connect to the bundle GitLab 14.6 ships with Mattermost 6.1 including potentially long running database migrations for Mattermost 6.0. For information about upgrading and for ways to reduce the downtime caused by those migrations, read the [Important Upgrade Notes](https://docs.mattermost.com/administration/important-upgrade-notes.html) for both versions. If you need to perform any manual migrations, [connect to the bundled PostgreSQL database](#connecting-to-the-bundled-postgresql-database). NOTE: -The Mattermost upgrade notes refer to different impacts when used with a PostgreSQL versus a MySQL database. The GitLab Mattermost included with the GitLab Linux packages uses a PostgreSQL database. +The Mattermost upgrade notes refer to different impacts when used with a PostgreSQL versus a MySQL database. The GitLab Mattermost included with the Linux package uses a PostgreSQL database. ## Upgrading GitLab Mattermost from versions prior to 11.0 @@ -492,7 +490,7 @@ If you encounter any issues [visit the GitLab Mattermost troubleshooting forum]( ### Upgrading GitLab Mattermost outside of GitLab -If you choose to upgrade Mattermost outside of the Omnibus GitLab automation, [follow this guide](https://docs.mattermost.com/administration/upgrade.html). +If you choose to upgrade Mattermost outside of the Linux package automation, [follow this guide](https://docs.mattermost.com/administration/upgrade.html). ## OAuth 2.0 sequence diagram diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index eed9f006bfa..7970f9711e6 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -53,13 +53,13 @@ cases you must consider. Follow the between versions. NOTE: -Version specific changes in Omnibus GitLab Linux packages can be found in [the Omnibus GitLab documentation](../update/package/index.md#version-specific-changes). +Version specific changes in Linux packages can be found in [the Linux package documentation](../update/package/index.md#version-specific-changes). NOTE: -Instructions are available for downloading an Omnibus GitLab Linux package locally and [manually installing](../update/package/index.md#upgrade-using-a-manually-downloaded-package) it. +Instructions are available for downloading the Linux package locally and [manually installing](../update/package/index.md#upgrade-using-a-manually-downloaded-package) it. NOTE: -A step-by-step guide to [upgrading the Omnibus-bundled PostgreSQL is documented separately](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server). +A step-by-step guide to [upgrading the Linux package-bundled PostgreSQL is documented separately](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server). ## Upgrading major versions diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 0d9e399f25c..0483dc4a3bf 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -259,7 +259,8 @@ are deprecated and will be removed from the GraphQL API. For installation instru </div> The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and Kubernetes Vanilla Operator to install a runner on Kubernetes is deprecated. Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, -will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: +will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). +The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` @@ -419,7 +420,9 @@ While the above approach is recommended for most instances, Sidekiq can also be The support for registration tokens and certain runner configuration arguments in the `POST` method operation on the `/api/v4/runners` endpoint is deprecated. This endpoint [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner with a GitLab instance at the instance, group, or project level through the API. Registration tokens, and support for certain configuration arguments, -will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: +will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). + +The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` @@ -446,7 +449,8 @@ This change is a breaking change. You should [create a runner in the UI](../ci/r Registration tokens and certain configuration arguments in the command `gitlab-runner register` that [registers](https://docs.gitlab.com/runner/register/) a runner, are deprecated. Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, -will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: +will be removed in GitLab 17.0. For more information, see [Migrating to the new runner registration workflow](../ci/runners/new_creation_workflow.md). +The configuration arguments disabled for authentication tokens are: - `--locked` - `--access-level` diff --git a/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb new file mode 100644 index 00000000000..68cc650d130 --- /dev/null +++ b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # It seeks duplicate npm packages and mark them for destruction + class MarkDuplicateNpmPackagesForDestruction < BatchedMigrationJob + NPM_PACKAGE_TYPE = 2 + PENDING_DESTRUCTION_STATUS = 4 + + operation_name :update_all + feature_category :package_registry + + # Temporary class to link AR model to the `packages_packages` table + class Package < ::ApplicationRecord + include EachBatch + + self.table_name = 'packages_packages' + end + + def perform + distinct_each_batch do |batch| + project_ids = batch.pluck(:project_id) + + subquery = Package + .where(project_id: project_ids, package_type: NPM_PACKAGE_TYPE) + .where.not(status: PENDING_DESTRUCTION_STATUS) + .select('project_id, name, version, MAX(id) AS max_id') + .group(:project_id, :name, :version) + .having('COUNT(*) > 1') + + join_query = <<~SQL.squish + INNER JOIN (#{subquery.to_sql}) AS duplicates + ON packages_packages.project_id = duplicates.project_id + AND packages_packages.name = duplicates.name + AND packages_packages.version = duplicates.version + SQL + + Package + .joins(join_query) + .where.not('packages_packages.id = duplicates.max_id') + .each_batch do |batch_to_update| + batch_to_update.update_all(status: PENDING_DESTRUCTION_STATUS) + end + end + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 115541c4914..9eeea7336b5 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -55,6 +55,7 @@ module Gitlab gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled if current_user + gon.version = Gitlab::VERSION # publish version only for logged in users gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name diff --git a/qa/Gemfile b/qa/Gemfile index fbd732327c0..2e7c9222695 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -6,7 +6,7 @@ gem 'gitlab-qa', '~> 11', '>= 11.2.0', require: 'gitlab/qa' gem 'gitlab_quality-test_tooling', '~> 0.6.2', require: false gem 'activesupport', '~> 6.1.7.2' # This should stay in sync with the root's Gemfile gem 'allure-rspec', '~> 2.20.0' -gem 'capybara', '~> 3.39.1' +gem 'capybara', '~> 3.39.2' gem 'capybara-screenshot', '~> 1.0.26' gem 'rake', '~> 13', '>= 13.0.6' gem 'rspec', '~> 3.12' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 7c6758982ae..8a628290c65 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -29,7 +29,7 @@ GEM debug_inspector (>= 0.0.1) builder (3.2.4) byebug (11.1.3) - capybara (3.39.1) + capybara (3.39.2) addressable matrix mini_mime (>= 0.1.3) @@ -313,7 +313,7 @@ DEPENDENCIES activesupport (~> 6.1.7.2) airborne (~> 0.3.7) allure-rspec (~> 2.20.0) - capybara (~> 3.39.1) + capybara (~> 3.39.2) capybara-screenshot (~> 1.0.26) chemlab (~> 0.10) chemlab-library-www-gitlab-com (~> 0.1, >= 0.1.1) diff --git a/spec/controllers/graphql_controller_spec.rb b/spec/controllers/graphql_controller_spec.rb index b1399835b45..b4a7e41ccd2 100644 --- a/spec/controllers/graphql_controller_spec.rb +++ b/spec/controllers/graphql_controller_spec.rb @@ -462,12 +462,13 @@ RSpec.describe GraphqlController, feature_category: :integrations do end it 'logs that it will try to hit the cache' do + expect(Gitlab::AppLogger).to receive(:info).with(message: "IntrospectionQueryCache hit") expect(Gitlab::AppLogger).to receive(:info).with( message: "IntrospectionQueryCache", - can_use_introspection_query_cache: true, + can_use_introspection_query_cache: "true", query: query.to_s, - variables: {}, - introspection_query_cache_key: ['introspection-query-cache', Gitlab.revision, false] + variables: "{}", + introspection_query_cache_key: "[\"introspection-query-cache\", \"#{Gitlab.revision}\", false]" ) post :execute, params: { query: query, operationName: 'IntrospectionQuery' } @@ -477,12 +478,13 @@ RSpec.describe GraphqlController, feature_category: :integrations do let(:query) { File.read(Rails.root.join('spec/fixtures/api/graphql/fake_introspection.graphql')) } it 'logs that it did not try to hit the cache' do + expect(Gitlab::AppLogger).to receive(:info).with(message: "IntrospectionQueryCache miss") expect(Gitlab::AppLogger).to receive(:info).with( message: "IntrospectionQueryCache", - can_use_introspection_query_cache: false, + can_use_introspection_query_cache: "false", query: query.to_s, - variables: {}, - introspection_query_cache_key: ['introspection-query-cache', Gitlab.revision, false] + variables: "{}", + introspection_query_cache_key: "[\"introspection-query-cache\", \"#{Gitlab.revision}\", false]" ) post :execute, params: { query: query, operationName: 'IntrospectionQuery' } diff --git a/spec/features/groups/new_group_page_spec.rb b/spec/features/groups/new_group_page_spec.rb index a73dbf998c3..c3731565ddf 100644 --- a/spec/features/groups/new_group_page_spec.rb +++ b/spec/features/groups/new_group_page_spec.rb @@ -11,24 +11,6 @@ RSpec.describe 'New group page', :js, feature_category: :groups_and_projects do sign_in(user) end - describe 'new top level group alert' do - context 'when a user visits the new group page' do - it 'shows the new top level group alert' do - visit new_group_path(anchor: 'create-group-pane') - - expect(page).to have_selector('[data-testid="new-top-level-alert"]') - end - end - - context 'when a user visits the new sub group page' do - it 'does not show the new top level group alert' do - visit new_group_path(parent_id: parent_group.id, anchor: 'create-group-pane') - - expect(page).not_to have_selector('[data-testid="new-top-level-alert"]') - end - end - end - describe 'sidebar' do context 'in the current navigation' do before do diff --git a/spec/frontend/sentry/index_spec.js b/spec/frontend/sentry/index_spec.js index aa19bb03cda..3130e01cc9e 100644 --- a/spec/frontend/sentry/index_spec.js +++ b/spec/frontend/sentry/index_spec.js @@ -4,6 +4,7 @@ import LegacySentryConfig from '~/sentry/legacy_sentry_config'; import SentryConfig from '~/sentry/sentry_config'; describe('Sentry init', () => { + const version = '1.0.0'; const dsn = 'https://123@sentry.gitlab.test/123'; const environment = 'test'; const currentUserId = '1'; @@ -13,6 +14,7 @@ describe('Sentry init', () => { beforeEach(() => { window.gon = { + version, sentry_dsn: dsn, sentry_environment: environment, current_user_id: currentUserId, @@ -42,7 +44,7 @@ describe('Sentry init', () => { currentUserId, allowUrls: [gitlabUrl, 'webpack-internal://'], environment, - release: revision, + release: version, tags: { revision, feature_category: featureCategory, diff --git a/spec/frontend/vue_shared/new_namespace/new_namespace_page_spec.js b/spec/frontend/vue_shared/new_namespace/new_namespace_page_spec.js index b87ae8a232f..1ee6f2cfff3 100644 --- a/spec/frontend/vue_shared/new_namespace/new_namespace_page_spec.js +++ b/spec/frontend/vue_shared/new_namespace/new_namespace_page_spec.js @@ -4,6 +4,7 @@ import { nextTick } from 'vue'; import LegacyContainer from '~/vue_shared/new_namespace/components/legacy_container.vue'; import WelcomePage from '~/vue_shared/new_namespace/components/welcome.vue'; import NewNamespacePage from '~/vue_shared/new_namespace/new_namespace_page.vue'; +import NewTopLevelGroupAlert from '~/groups/components/new_top_level_group_alert.vue'; import SuperSidebarToggle from '~/super_sidebar/components/super_sidebar_toggle.vue'; import { sidebarState } from '~/super_sidebar/constants'; @@ -14,6 +15,7 @@ describe('Experimental new namespace creation app', () => { const findWelcomePage = () => wrapper.findComponent(WelcomePage); const findLegacyContainer = () => wrapper.findComponent(LegacyContainer); const findBreadcrumb = () => wrapper.findComponent(GlBreadcrumb); + const findNewTopLevelGroupAlert = () => wrapper.findComponent(NewTopLevelGroupAlert); const findSuperSidebarToggle = () => wrapper.findComponent(SuperSidebarToggle); const DEFAULT_PROPS = { @@ -33,6 +35,9 @@ describe('Experimental new namespace creation app', () => { ...DEFAULT_PROPS, ...propsData, }, + stubs: { + NewTopLevelGroupAlert, + }, }); }; @@ -125,4 +130,39 @@ describe('Experimental new namespace creation app', () => { expect(findSuperSidebarToggle().exists()).toBe(isToggleVisible); }); }); + + describe('top level group alert', () => { + beforeEach(() => { + window.location.hash = `#${DEFAULT_PROPS.panels[0].name}`; + }); + + describe('when self-managed', () => { + it('does not render alert', () => { + createComponent(); + + expect(findNewTopLevelGroupAlert().exists()).toBe(false); + }); + }); + + describe('when on .com', () => { + it('does not render alert', () => { + createComponent({ propsData: { isSaas: true } }); + + expect(findNewTopLevelGroupAlert().exists()).toBe(false); + }); + + describe('when empty parent group name', () => { + it('renders alert', () => { + createComponent({ + propsData: { + isSaas: true, + panels: [{ ...DEFAULT_PROPS.panels[0], detailProps: { parentGroupName: '' } }], + }, + }); + + expect(findNewTopLevelGroupAlert().exists()).toBe(true); + }); + }); + }); + }); }); diff --git a/spec/helpers/groups_helper_spec.rb b/spec/helpers/groups_helper_spec.rb index f66f9a8a58e..bdcf0ef57ee 100644 --- a/spec/helpers/groups_helper_spec.rb +++ b/spec/helpers/groups_helper_spec.rb @@ -437,7 +437,8 @@ RSpec.describe GroupsHelper do expect(subgroup_creation_data(subgroup)).to eq({ import_existing_group_path: '/groups/new#import-group-pane', parent_group_name: name, - parent_group_url: group_url(group) + parent_group_url: group_url(group), + is_saas: 'false' }) end end @@ -447,7 +448,8 @@ RSpec.describe GroupsHelper do expect(subgroup_creation_data(group)).to eq({ import_existing_group_path: '/groups/new#import-group-pane', parent_group_name: nil, - parent_group_url: nil + parent_group_url: nil, + is_saas: 'false' }) end end diff --git a/spec/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction_spec.rb b/spec/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction_spec.rb new file mode 100644 index 00000000000..05a19b7973c --- /dev/null +++ b/spec/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MarkDuplicateNpmPackagesForDestruction, schema: 20230524201454, feature_category: :package_registry do # rubocop:disable Layout/LineLength + describe '#perform' do + let(:projects_table) { table(:projects) } + let(:namespaces_table) { table(:namespaces) } + let(:packages_table) { table(:packages_packages) } + + let!(:namespace) do + namespaces_table.create!(name: 'project', path: 'project', type: 'Project') + end + + let!(:project) do + projects_table.create!( + namespace_id: namespace.id, + name: 'project', + path: 'project', + project_namespace_id: namespace.id + ) + end + + let!(:package_1) do + packages_table.create!( + project_id: project.id, + name: 'test1', + version: '1.0.0', + package_type: described_class::NPM_PACKAGE_TYPE + ) + end + + let!(:package_2) do + packages_table.create!( + project_id: project.id, + name: 'test2', + version: '1.0.0', + package_type: described_class::NPM_PACKAGE_TYPE + ) + end + + let!(:package_3) do + packages_table.create!( + project_id: project.id, + name: 'test3', + version: '1.0.0', + package_type: described_class::NPM_PACKAGE_TYPE + ) + end + + let(:migration) do + described_class.new( + start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :packages_packages, + batch_column: :project_id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + before do + # create a duplicated package without triggering model validation errors + package_2.update_column(:name, package_1.name) + package_3.update_column(:name, package_1.name) + end + + it 'marks duplicate npm packages for destruction', :aggregate_failures do + packages_marked_for_destruction = described_class::Package + .where(status: described_class::PENDING_DESTRUCTION_STATUS) + + expect { migration.perform } + .to change { packages_marked_for_destruction.count }.from(0).to(2) + expect(package_3.reload.status).not_to eq(described_class::PENDING_DESTRUCTION_STATUS) + end + end +end diff --git a/spec/lib/gitlab/gon_helper_spec.rb b/spec/lib/gitlab/gon_helper_spec.rb index 6e8997d51c3..1135cfc22ac 100644 --- a/spec/lib/gitlab/gon_helper_spec.rb +++ b/spec/lib/gitlab/gon_helper_spec.rb @@ -6,10 +6,6 @@ RSpec.describe Gitlab::GonHelper do let(:helper) do Class.new do include Gitlab::GonHelper - - def current_user - nil - end end.new end @@ -18,6 +14,7 @@ RSpec.describe Gitlab::GonHelper do let(:https) { true } before do + allow(helper).to receive(:current_user).and_return(nil) allow(helper).to receive(:gon).and_return(gon) stub_config_setting(https: https) end @@ -40,6 +37,24 @@ RSpec.describe Gitlab::GonHelper do end end + it 'sets no GitLab version' do + expect(gon).not_to receive(:version=) + + helper.add_gon_variables + end + + context 'when user is logged in' do + before do + allow(helper).to receive(:current_user).and_return(build_stubbed(:user)) + end + + it 'sets GitLab version' do + expect(gon).to receive(:version=).with(Gitlab::VERSION) + + helper.add_gon_variables + end + end + context 'when sentry is configured' do let(:clientside_dsn) { 'https://xxx@sentry.example.com/1' } let(:environment) { 'staging' } diff --git a/spec/migrations/20230524201454_queue_mark_duplicate_npm_packages_for_destruction_spec.rb b/spec/migrations/20230524201454_queue_mark_duplicate_npm_packages_for_destruction_spec.rb new file mode 100644 index 00000000000..639c84e9bec --- /dev/null +++ b/spec/migrations/20230524201454_queue_mark_duplicate_npm_packages_for_destruction_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueMarkDuplicateNpmPackagesForDestruction, feature_category: :package_registry do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :packages_packages, + column_name: :project_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + batch_class_name: described_class::BATCH_CLASS_NAME, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/spec/models/integrations/hangouts_chat_spec.rb b/spec/models/integrations/hangouts_chat_spec.rb index 1ebf2ec3005..bcb80768ffb 100644 --- a/spec/models/integrations/hangouts_chat_spec.rb +++ b/spec/models/integrations/hangouts_chat_spec.rb @@ -2,7 +2,7 @@ require "spec_helper" -RSpec.describe Integrations::HangoutsChat do +RSpec.describe Integrations::HangoutsChat, feature_category: :integrations do it_behaves_like "chat integration", "Hangouts Chat" do let(:client) { HangoutsChat::Sender } let(:client_arguments) { webhook_url } @@ -46,25 +46,27 @@ RSpec.describe Integrations::HangoutsChat do end context 'with issue events' do - let(:issues_sample_data) { create(:issue, project: project).to_hook_data(user) } + let_it_be(:issue) { create(:issue, project: project) } + let(:issues_sample_data) { issue.to_hook_data(user) } it "adds thread key for issue events" do expect(chat_integration.execute(issues_sample_data)).to be(true) expect(WebMock).to have_requested(:post, webhook_url) - .with(query: hash_including({ "threadKey" => /issue .*?/ })) + .with(query: hash_including({ "threadKey" => /issue #{project.full_name}##{issue.iid}/ })) .once end end context 'with merge events' do - let(:merge_sample_data) { create(:merge_request, source_project: project).to_hook_data(user) } + let_it_be(:merge_request) { create(:merge_request, source_project: project) } + let(:merge_sample_data) { merge_request.to_hook_data(user) } it "adds thread key for merge events" do expect(chat_integration.execute(merge_sample_data)).to be(true) expect(WebMock).to have_requested(:post, webhook_url) - .with(query: hash_including({ "threadKey" => /merge request .*?/ })) + .with(query: hash_including({ "threadKey" => /merge request #{project.full_name}!#{merge_request.iid}/ })) .once end end |