From 91a7f1897e7cb14d60c2aaf3afa76ebdc879d939 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 25 May 2023 09:09:38 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .gitlab/ci/global.gitlab-ci.yml | 24 ++ .gitlab/ci/rails.gitlab-ci.yml | 36 +++ .gitlab/ci/rails/shared.gitlab-ci.yml | 4 + .gitlab/ci/rules.gitlab-ci.yml | 27 +++ .rubocop_todo/layout/argument_alignment.yml | 1 - .rubocop_todo/rspec/factory_bot/avoid_create.yml | 2 +- .rubocop_todo/rspec/missing_feature_category.yml | 1 - app/services/bulk_imports/create_service.rb | 5 +- config/redis.yml.example | 10 + config/routes.rb | 7 +- doc/api/graphql/getting_started.md | 2 +- doc/integration/advanced_search/elasticsearch.md | 3 +- doc/user/project/integrations/hangouts_chat.md | 4 + lib/bulk_imports/clients/http.rb | 4 +- .../saas/container_registry_spec.rb | 122 ++++++++++ .../api/5_package/container_registry_spec.rb | 123 ---------- .../container_registry_omnibus_spec.rb | 259 -------------------- .../container_registry/container_registry_spec.rb | 92 ------- .../online_garbage_collection_spec.rb | 112 --------- .../saas/container_registry_spec.rb | 94 ++++++++ .../self_managed/container_registry_spec.rb | 267 +++++++++++++++++++++ .../dependency_proxy/dependency_proxy_spec.rb | 56 +---- scripts/prepare_build.sh | 7 + .../controllers/admin/dashboard_controller_spec.rb | 3 +- spec/lib/bulk_imports/clients/http_spec.rb | 14 +- spec/support/rspec_order_todo.yml | 4 +- 26 files changed, 632 insertions(+), 651 deletions(-) create mode 100644 config/redis.yml.example create mode 100644 qa/qa/specs/features/api/5_package/container_registry/saas/container_registry_spec.rb delete mode 100644 qa/qa/specs/features/api/5_package/container_registry_spec.rb delete mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb delete mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb delete mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb create mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/saas/container_registry_spec.rb create mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 4362269103b..7d359dbc5c9 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -222,6 +222,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-12-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine variables: POSTGRES_HOST_AUTH_METHOD: trust @@ -232,6 +234,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-13-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.2-alpine variables: POSTGRES_HOST_AUTH_METHOD: trust @@ -242,6 +246,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.2-alpine variables: POSTGRES_HOST_AUTH_METHOD: trust @@ -252,6 +258,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-12-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: elasticsearch:7.17.6 command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] @@ -268,6 +276,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-13-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.2-alpine - name: elasticsearch:7.17.6 command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] @@ -284,6 +294,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.2-alpine - name: elasticsearch:7.17.6 command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] @@ -300,6 +312,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-13-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: elasticsearch:8.6.2 - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0 @@ -317,6 +331,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: elasticsearch:8.6.2 - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.0 @@ -334,6 +350,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-13-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: opensearchproject/opensearch:1.3.5 alias: elasticsearch @@ -351,6 +369,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-13-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: opensearchproject/opensearch:2.2.1 alias: elasticsearch @@ -368,6 +388,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: opensearchproject/opensearch:1.3.5 alias: elasticsearch @@ -385,6 +407,8 @@ - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-14-pgvector-0.4.1 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] alias: postgres + - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12 + alias: rediscluster # configure connections in config/redis.yml - name: redis:6.0-alpine - name: opensearchproject/opensearch:2.2.1 alias: elasticsearch diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index a4041f771d9..d2820149102 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -120,6 +120,12 @@ rspec unit pg13: - .rails:rules:ee-and-foss-unit - .rspec-unit-parallel +rspec unit pg13 single-redis: + extends: + - rspec unit pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec unit pg13 single-db: extends: - rspec unit pg13 @@ -144,6 +150,12 @@ rspec integration pg13: - .rails:rules:ee-and-foss-integration - .rspec-integration-parallel +rspec integration pg13 single-redis: + extends: + - rspec integration pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec integration pg13 single-db: extends: - rspec integration pg13 @@ -170,6 +182,12 @@ rspec system pg13: variables: DEBUG_GITLAB_TRANSACTION_STACK: "true" +rspec system pg13 single-redis: + extends: + - rspec system pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec system pg13 single-db: extends: - rspec system pg13 @@ -572,6 +590,12 @@ rspec-ee unit pg13 single-db: - .single-db-rspec - .rails:rules:single-db +rspec-ee unit pg13 single-redis: + extends: + - rspec-ee unit pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec-ee unit pg13 single-db-ci-connection: extends: - rspec-ee unit pg13 @@ -595,6 +619,12 @@ rspec-ee integration pg13 single-db: - .single-db-rspec - .rails:rules:single-db +rspec-ee integration pg13 single-redis: + extends: + - rspec-ee integration pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec-ee integration pg13 single-db-ci-connection: extends: - rspec-ee integration pg13 @@ -618,6 +648,12 @@ rspec-ee system pg13 single-db: - .single-db-rspec - .rails:rules:single-db +rspec-ee system pg13 single-redis: + extends: + - rspec-ee system pg13 + - .no-redis-cluster + - .rails:rules:single-redis + rspec-ee system pg13 single-db-ci-connection: extends: - rspec-ee system pg13 diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index 33aef166afd..47226032d4f 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -28,6 +28,10 @@ include: - section_start "gitaly-test-spawn" "Spawning Gitaly"; scripts/gitaly-test-spawn; section_end "gitaly-test-spawn" # Do not use 'bundle exec' here - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-spawn\r\e[0K" +.no-redis-cluster: + variables: + USE_REDIS_CLUSTER: "false" + .single-db: variables: DECOMPOSED_DB: "false" diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index a37d0e2be37..e76c127d420 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -338,6 +338,24 @@ - "{,ee/,jh/}{bin,config,db,generator_templates,lib}/**/*" - "{,ee/,jh/}spec/**/*" +# Redis patterns + feature flags +.redis-patterns: &redis-patterns + - "{,ee/,jh/}{,spec/}app/models/concerns/redis_cacheable{,_spec}.rb" + - "{,ee/,jh/}{,spec/}app/models/ci/build_trace_chunks/redis{,_base,_trace_chunks}{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/usage_data_counters/{hll_redis_counter,redis_counter}{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/usage/metrics/instrumentations/redis{_metric,hll_metric}{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/usage/metrics/aggregates/sources/redis_hll{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/patch/action_cable_redis_listener{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/merge_requests/mergeability/redis_interface{,_spec}.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/markdown_cache/redis/*.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/redis/**/*.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/instrumentation/*.rb" + - "{,ee/,jh/}{,spec/}lib/{,ee/,jh/}gitlab/redis.rb" + - "{,ee/,jh/}config/initializers/7_redis.rb" + # Feature flag changes + - "{,ee/,jh/}{,spec/}lib/feature.rb" + - "config/feature_flags/**/*" + # DB patterns + .ci-patterns .db-patterns: &db-patterns - "{,ee/,jh/}{,spec/}{db,migrations}/**/*" @@ -1567,6 +1585,15 @@ changes: *setup-test-env-patterns - <<: *if-merge-request-labels-run-all-rspec +.rails:rules:single-redis: + rules: + - <<: *if-merge-request-labels-run-single-db + - <<: *if-merge-request-labels-pipeline-expedite + when: never + - <<: *if-merge-request + changes: *redis-patterns + - <<: *if-default-branch-schedule-nightly + .rails:rules:single-db: rules: - <<: *if-merge-request-labels-run-single-db diff --git a/.rubocop_todo/layout/argument_alignment.yml b/.rubocop_todo/layout/argument_alignment.yml index 506fe38ab51..c78390f4874 100644 --- a/.rubocop_todo/layout/argument_alignment.yml +++ b/.rubocop_todo/layout/argument_alignment.yml @@ -1359,7 +1359,6 @@ Layout/ArgumentAlignment: - 'ee/spec/requests/groups/protected_environments_controller_spec.rb' - 'ee/spec/requests/groups/settings/domain_verification_controller_spec.rb' - 'ee/spec/requests/groups/two_factor_auths_controller_spec.rb' - - 'ee/spec/requests/registrations/project_creation_spec.rb' - 'ee/spec/requests/smartcard_controller_spec.rb' - 'ee/spec/requests/users/identity_verification_controller_spec.rb' - 'ee/spec/services/analytics/cycle_analytics/aggregator_service_spec.rb' diff --git a/.rubocop_todo/rspec/factory_bot/avoid_create.yml b/.rubocop_todo/rspec/factory_bot/avoid_create.yml index 3a990dcd12d..e994e3e516c 100644 --- a/.rubocop_todo/rspec/factory_bot/avoid_create.yml +++ b/.rubocop_todo/rspec/factory_bot/avoid_create.yml @@ -224,7 +224,7 @@ RSpec/FactoryBot/AvoidCreate: - 'ee/spec/views/projects/security/policies/index.html.haml_spec.rb' - 'ee/spec/views/projects/security/sast_configuration/show.html.haml_spec.rb' - 'ee/spec/views/projects/settings/subscriptions/_index.html.haml_spec.rb' - - 'ee/spec/views/registrations/groups_projects/new.html.haml_spec.rb' + - 'ee/spec/views/registrations/groups/new.html.haml_spec.rb' - 'ee/spec/views/shared/_clone_panel.html.haml_spec.rb' - 'ee/spec/views/shared/_kerberos_clone_button.html.haml_spec.rb' - 'ee/spec/views/shared/_mirror_status.html.haml_spec.rb' diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml index f4ecd85bd47..7a60575ebf2 100644 --- a/.rubocop_todo/rspec/missing_feature_category.yml +++ b/.rubocop_todo/rspec/missing_feature_category.yml @@ -1573,7 +1573,6 @@ RSpec/MissingFeatureCategory: - 'ee/spec/views/projects/security/sast_configuration/show.html.haml_spec.rb' - 'ee/spec/views/projects/settings/merge_requests/_merge_request_approvals.html.haml_spec.rb' - 'ee/spec/views/projects/settings/subscriptions/_index.html.haml_spec.rb' - - 'ee/spec/views/registrations/groups_projects/new.html.haml_spec.rb' - 'ee/spec/views/shared/_clone_panel.html.haml_spec.rb' - 'ee/spec/views/shared/_kerberos_clone_button.html.haml_spec.rb' - 'ee/spec/views/shared/_mirror_status.html.haml_spec.rb' diff --git a/app/services/bulk_imports/create_service.rb b/app/services/bulk_imports/create_service.rb index 4c9c59ac504..636c636255f 100644 --- a/app/services/bulk_imports/create_service.rb +++ b/app/services/bulk_imports/create_service.rb @@ -118,9 +118,10 @@ module BulkImports end client.get("/#{entity_type}/#{source_entity_identifier}/export_relations/status") + rescue BulkImports::NetworkError => e # the source instance will return a 404 if the feature is disabled as the endpoint won't be available - rescue Gitlab::HTTP::BlockedUrlError - rescue BulkImports::NetworkError + return if e.cause.is_a?(Gitlab::HTTP::BlockedUrlError) + raise ::BulkImports::Error.setting_not_enabled end diff --git a/config/redis.yml.example b/config/redis.yml.example new file mode 100644 index 00000000000..a2dca88491b --- /dev/null +++ b/config/redis.yml.example @@ -0,0 +1,10 @@ +--- +development: + rate_limiting: + cluster: + - redis://localhost:7001 + +test: + rate_limiting: + cluster: + - redis://localhost:7001 diff --git a/config/routes.rb b/config/routes.rb index 9c8ad8fe047..575b3089307 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -66,7 +66,12 @@ InitializerConnections.raise_if_new_database_connection do Gitlab.ee do resource :company, only: [:new, :create], controller: 'company' - resources :groups_projects, only: [:new, :create] do + # TODO: remove next line and the controller after the deployment + # https://gitlab.com/gitlab-org/gitlab/-/issues/411208 + resources :groups_projects, only: [:create] do + post :import, on: :collection + end + resources :groups, only: [:new, :create] do post :import, on: :collection end draw :verification diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index 237c0cc6934..5142496753c 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -4,7 +4,7 @@ group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Get started with GitLab GraphQL API **(FREE)** +# Run GraphQL API queries and mutations **(FREE)** This guide demonstrates basic usage of the GitLab GraphQL API. diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index e886f2a4b37..71268e8e78e 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -219,8 +219,7 @@ The following Elasticsearch settings are available: | `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://:@:9200/`. | | `Username` | The `username` of your Elasticsearch instance. | | `Password` | The password of your Elasticsearch instance. | -| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | -| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | +| `Number of Elasticsearch shards and replicas per index` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least five shards. Indices with tens of millions of documents should have more shards ([see the guidance](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until you re-create the index. For more information about scalability and resilience, see the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). Each Elasticsearch shard can have a number of replicas. These replicas are a complete copy of the shard and can provide increased query performance or resilience against hardware failure. Increasing this value increases the total disk space required by the index. You can set the number of shards and replicas for each of the indices. | | `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).| | `Using AWS OpenSearch Service with IAM credentials` | Sign your OpenSearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | | `AWS Region` | The AWS region in which your OpenSearch Service is located. | diff --git a/doc/user/project/integrations/hangouts_chat.md b/doc/user/project/integrations/hangouts_chat.md index 9f47b531f36..1cfe1a02692 100644 --- a/doc/user/project/integrations/hangouts_chat.md +++ b/doc/user/project/integrations/hangouts_chat.md @@ -40,6 +40,10 @@ For further details, see [the Google Chat documentation for configuring webhooks > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27823) in GitLab 15.4. +WARNING: +In March 2023, Google [deprecated threaded replies in Google Chat](https://workspaceupdates.googleblog.com/2023/02/new-google-chat-spaces-will-be-in-line-threaded.html). +This feature does not work for new Google Chat spaces. You can still use this feature in existing Google Chat spaces where threaded replies are already enabled. + To enable threaded notifications for the same GitLab object (for example, an issue or merge request): 1. Go to [Google Chat](https://chat.google.com/). diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 616ab8754b4..c9ed75e663e 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -158,15 +158,13 @@ module BulkImports { timeout: SIDEKIQ_REQUEST_TIMEOUT } if Gitlab::Runtime.sidekiq? end + # @raise [BulkImports::NetworkError] when unsuccessful def with_error_handling response = yield return response if response.success? raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response) - - rescue Gitlab::HTTP::BlockedUrlError => e - raise e rescue *Gitlab::HTTP::HTTP_ERRORS => e raise ::BulkImports::NetworkError, e end diff --git a/qa/qa/specs/features/api/5_package/container_registry/saas/container_registry_spec.rb b/qa/qa/specs/features/api/5_package/container_registry/saas/container_registry_spec.rb new file mode 100644 index 00000000000..ffbe31ed025 --- /dev/null +++ b/qa/qa/specs/features/api/5_package/container_registry/saas/container_registry_spec.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +require 'airborne' + +module QA + RSpec.describe 'Package', :reliable, only: { subdomain: %i[staging staging-canary pre] }, + product_group: :container_registry do + include Support::API + include Support::Helpers::MaskToken + + describe 'SaaS Container Registry API' do + let(:api_client) { Runtime::API::Client.new(:gitlab) } + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-registry-api' + project.template_name = 'express' + project.api_client = api_client + end + end + + let!(:project_access_token) do + QA::Resource::ProjectAccessToken.fabricate_via_api! do |pat| + pat.project = project + end + end + + let(:registry) do + Resource::RegistryRepository.init do |repository| + repository.name = project.path_with_namespace + repository.project = project + repository.tag_name = 'master' + end + end + + let(:masked_token) do + use_ci_variable(name: 'PAT', value: project_access_token.token, project: project) + end + + let(:gitlab_ci_yaml) do + <<~YAML + stages: + - build + - test + + build: + image: docker:24.0.1 + stage: build + services: + - docker:24.0.1-dind + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + before_script: + - until docker info; do sleep 1; done + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + - docker pull $IMAGE_TAG + + test: + image: dwdraju/alpine-curl-jq:latest + stage: test + script: + - 'id=$(curl --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories" | jq ".[0].id")' + - echo $id + - 'tag_count=$(curl --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags" | jq ". | length")' + - if [ $tag_count -ne 1 ]; then exit 1; fi; + - 'status_code=$(curl --request DELETE --head --output /dev/null --write-out "%{http_code}\n" --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags/master")' + - if [ $status_code -ne 200 ]; then exit 1; fi; + - 'status_code=$(curl --head --output /dev/null --write-out "%{http_code}\n" --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags/master")' + - if [ $status_code -ne 404 ]; then exit 1; fi; + YAML + end + + it 'pushes, pulls image to the registry and deletes tag', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348001' do + Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2, message: "Commit push") do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.api_client = api_client + commit.commit_message = 'Add .gitlab-ci.yml' + commit.project = project + commit.add_files([{ + file_path: '.gitlab-ci.yml', + content: gitlab_ci_yaml + }]) + end + end + + Support::Retrier.retry_until( + max_duration: 10, + sleep_interval: 1, + message: "Waiting for pipeline to start" + ) do + pipeline_is_triggered? + end + Support::Retrier.retry_until( + max_duration: 300, + sleep_interval: 5, + message: "Waiting for pipeline to succeed" + ) do + latest_pipeline_succeed? + end + end + + private + + def pipeline_is_triggered? + !project.pipelines.empty? + end + + def latest_pipeline_succeed? + latest_pipeline = project.pipelines.first + latest_pipeline[:status] == 'success' + end + end + end +end diff --git a/qa/qa/specs/features/api/5_package/container_registry_spec.rb b/qa/qa/specs/features/api/5_package/container_registry_spec.rb deleted file mode 100644 index a0d69ab5654..00000000000 --- a/qa/qa/specs/features/api/5_package/container_registry_spec.rb +++ /dev/null @@ -1,123 +0,0 @@ -# frozen_string_literal: true - -require 'airborne' - -module QA - RSpec.describe 'Package', :reliable, product_group: :container_registry, only: { - subdomain: %i[staging staging-canary pre] - } do - include Support::API - include Support::Helpers::MaskToken - - describe 'Container Registry' do - let(:api_client) { Runtime::API::Client.new(:gitlab) } - - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'project-with-registry-api' - project.template_name = 'express' - project.api_client = api_client - end - end - - let!(:project_access_token) do - QA::Resource::ProjectAccessToken.fabricate_via_api! do |pat| - pat.project = project - end - end - - let(:registry) do - Resource::RegistryRepository.init do |repository| - repository.name = project.path_with_namespace - repository.project = project - repository.tag_name = 'master' - end - end - - let(:masked_token) do - use_ci_variable(name: 'PAT', value: project_access_token.token, project: project) - end - - let(:gitlab_ci_yaml) do - <<~YAML - stages: - - build - - test - - build: - image: docker:19.03.12 - stage: build - services: - - docker:19.03.12-dind - variables: - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - before_script: - - until docker info; do sleep 1; done - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker build -t $IMAGE_TAG . - - docker push $IMAGE_TAG - - docker pull $IMAGE_TAG - - test: - image: dwdraju/alpine-curl-jq:latest - stage: test - script: - - 'id=$(curl --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories" | jq ".[0].id")' - - echo $id - - 'tag_count=$(curl --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags" | jq ". | length")' - - if [ $tag_count -ne 1 ]; then exit 1; fi; - - 'status_code=$(curl --request DELETE --head --output /dev/null --write-out "%{http_code}\n" --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags/master")' - - if [ $status_code -ne 200 ]; then exit 1; fi; - - 'status_code=$(curl --head --output /dev/null --write-out "%{http_code}\n" --header "PRIVATE-TOKEN: #{masked_token}" "https://${CI_SERVER_HOST}/api/v4/projects/#{project.id}/registry/repositories/$id/tags/master")' - - if [ $status_code -ne 404 ]; then exit 1; fi; - YAML - end - - it 'pushes, pulls image to the registry and deletes tag', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348001' do - Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2, message: "Commit push") do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.api_client = api_client - commit.commit_message = 'Add .gitlab-ci.yml' - commit.project = project - commit.add_files([{ - file_path: '.gitlab-ci.yml', - content: gitlab_ci_yaml - }]) - end - end - - Support::Retrier.retry_until( - max_duration: 10, - sleep_interval: 1, - message: "Waiting for pipeline to start" - ) do - pipeline_is_triggered? - end - Support::Retrier.retry_until( - max_duration: 300, - sleep_interval: 5, - message: "Waiting for pipeline to succeed" - ) do - latest_pipeline_succeed? - end - end - - private - - def pipeline_is_triggered? - !project.pipelines.empty? - end - - def latest_pipeline_succeed? - latest_pipeline = project.pipelines.first - latest_pipeline[:status] == 'success' - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb deleted file mode 100644 index b39475b481d..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_omnibus_spec.rb +++ /dev/null @@ -1,259 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Package', :orchestrated, :skip_live_env, product_group: :container_registry do - describe 'Self-managed Container Registry' do - include Support::Helpers::MaskToken - - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'project-with-registry' - project.template_name = 'express' - project.visibility = :private - end - end - - let(:project_deploy_token) do - Resource::ProjectDeployToken.fabricate_via_api! do |deploy_token| - deploy_token.name = 'registry-deploy-token' - deploy_token.project = project - deploy_token.scopes = %w[ - read_repository - read_package_registry - write_package_registry - read_registry - write_registry - ] - end - end - - let!(:runner) do - Resource::ProjectRunner.fabricate! do |runner| - runner.name = "qa-runner-#{Time.now.to_i}" - runner.tags = ["runner-for-#{project.name}"] - runner.executor = :docker - runner.project = project - end - end - - let(:personal_access_token) { Runtime::Env.personal_access_token } - - before do - Flow::Login.sign_in - project.visit! - end - - after do - runner.remove_via_api! - end - - context "when tls is disabled" do - where do - { - 'using docker:18.09.9 and a personal access token' => { - docker_client_version: 'docker:18.09.9', - authentication_token_type: :personal_access_token, - token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348499' - }, - 'using docker:18.09.9 and a project deploy token' => { - docker_client_version: 'docker:18.09.9', - authentication_token_type: :project_deploy_token, - token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348852' - }, - 'using docker:18.09.9 and a ci job token' => { - docker_client_version: 'docker:18.09.9', - authentication_token_type: :ci_job_token, - token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348765' - }, - 'using docker:19.03.12 and a personal access token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :personal_access_token, - token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348507' - }, - 'using docker:19.03.12 and a project deploy token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :project_deploy_token, - token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348859' - }, - 'using docker:19.03.12 and a ci job token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :ci_job_token, - token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348654' - }, - 'using docker:20.10 and a personal access token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :personal_access_token, - token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348754' - }, - 'using docker:20.10 and a project deploy token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :project_deploy_token, - token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348856' - }, - 'using docker:20.10 and a ci job token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :ci_job_token, - token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/348766' - } - } - end - - with_them do - let(:auth_token) do - case authentication_token_type - when :personal_access_token - use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: project) - when :project_deploy_token - use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) - when :ci_job_token - '$CI_JOB_TOKEN' - end - end - - let(:auth_user) do - case authentication_token_type - when :personal_access_token - "$CI_REGISTRY_USER" - when :project_deploy_token - "\"#{project_deploy_token.username}\"" - when :ci_job_token - 'gitlab-ci-token' - end - end - - it "pushes image and deletes tag", :registry, testcase: params[:testcase] do - Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files( - [ - { - file_path: '.gitlab-ci.yml', - content: - <<~YAML - build: - image: "#{docker_client_version}" - stage: build - services: - - name: "#{docker_client_version}-dind" - command: ["--insecure-registry=gitlab.test:5050"] - variables: - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - script: - - docker login -u #{auth_user} -p #{auth_token} gitlab.test:5050 - - docker build -t $IMAGE_TAG . - - docker push $IMAGE_TAG - tags: - - "runner-for-#{project.name}" - YAML - } - ] - ) - end - end - - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - Page::Project::Menu.perform(&:go_to_container_registry) - - Page::Project::Registry::Show.perform do |registry| - expect(registry).to have_registry_repository(project.name) - - registry.click_on_image(project.name) - expect(registry).to have_tag('master') - end - end - end - end - - context 'when tls is enabled' do - it( - 'pushes image and deletes tag', - :registry_tls, - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347591' - ) do - Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files( - [ - { - file_path: '.gitlab-ci.yml', - content: - <<~YAML - build: - image: docker:19.03.12 - stage: build - services: - - name: docker:19.03.12-dind - command: - - /bin/sh - - -c - - | - apk add --no-cache openssl - true | openssl s_client -showcerts -connect gitlab.test:5050 > /usr/local/share/ca-certificates/gitlab.test.crt - update-ca-certificates - dockerd-entrypoint.sh || exit - variables: - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD gitlab.test:5050 - - docker build -t $IMAGE_TAG . - - docker push $IMAGE_TAG - tags: - - "runner-for-#{project.name}" - YAML - } - ] - ) - end - end - - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - - Support::Retrier.retry_until(max_duration: 800, sleep_interval: 10) do - project.pipelines.last[:status] == 'success' - end - - Page::Project::Menu.perform(&:go_to_container_registry) - - Page::Project::Registry::Show.perform do |registry| - expect(registry).to have_registry_repository(project.name) - - registry.click_on_image(project.name) - - expect(registry).to have_tag('master') - - registry.click_delete - - expect { registry.has_no_tag?('master') } - .to eventually_be_truthy.within(max_duration: 60, reload_page: page) - end - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb deleted file mode 100644 index 680b722edb7..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/container_registry_spec.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Package' do - describe 'Container Registry', only: { subdomain: %i[staging staging-canary pre] }, product_group: :container_registry do - let(:project) do - Resource::Project.fabricate_via_api! do |project| - project.name = 'project-with-registry' - project.template_name = 'express' - end - end - - let(:registry_repository) do - Resource::RegistryRepository.fabricate! do |repository| - repository.name = "#{project.path_with_namespace}" - repository.project = project - end - end - - let!(:gitlab_ci_yaml) do - <<~YAML - build: - image: docker:19.03.12 - stage: build - services: - - docker:19.03.12-dind - variables: - IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - before_script: - - | - echo "Waiting for docker to start..." - for i in $(seq 1 30) - do - docker info && break - sleep 1s - done - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker build -t $IMAGE_TAG . - - docker push $IMAGE_TAG - YAML - end - - after do - registry_repository&.remove_via_api! - end - - it 'pushes project image to the container registry and deletes tag', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347687' do - Flow::Login.sign_in - project.visit! - - Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = project - commit.commit_message = 'Add .gitlab-ci.yml' - commit.add_files([{ - file_path: '.gitlab-ci.yml', - content: gitlab_ci_yaml - }]) - end - end - - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('build') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 800) - end - - Page::Project::Menu.perform(&:go_to_container_registry) - - Page::Project::Registry::Show.perform do |registry| - expect(registry).to have_registry_repository(project.name) - - registry.click_on_image(project.name) - expect(registry).to have_tag('master') - - registry.click_delete - expect { registry.has_no_tag?('master') } - .to eventually_be_truthy.within(max_duration: 60, reload_page: page) - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb deleted file mode 100644 index 1e076117e4e..00000000000 --- a/qa/qa/specs/features/browser_ui/5_package/container_registry/online_garbage_collection_spec.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Package', :requires_admin, product_group: :container_registry do - describe 'Container Registry Online Garbage Collection', :registry_gc, only: { subdomain: %i[pre] } do - let(:group) { Resource::Group.fabricate_via_api! } - - let(:imported_project) do - Resource::ProjectImportedFromURL.fabricate_via_browser_ui! do |project| - project.name = 'container-registry' - project.group = group - project.gitlab_repository_path = 'https://gitlab.com/gitlab-org/container-registry.git' - end - end - - let!(:gitlab_ci_yaml) do - <<~YAML - variables: - GOPATH: $CI_PROJECT_DIR/.go - BUILD_CACHE: $CI_PROJECT_DIR/.online-gc-tester - STAGE_ONE_VALIDATION_DELAY: "6m" - STAGE_TWO_VALIDATION_DELAY: "12m" - STAGE_THREE_VALIDATION_DELAY: "6m" - STAGE_FOUR_VALIDATION_DELAY: "12m" - STAGE_FIVE_VALIDATION_DELAY: "12m" - - stages: - - generate - - build - - test - - .base: &base - image: docker:19 - services: - - docker:19-dind - variables: - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - before_script: - - until docker info; do sleep 1; done - - mkdir -p $GOPATH - - mkdir -p $BUILD_CACHE - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - test: - stage: generate - extends: .base - script: - - apk add go make git - - make binaries - - ./bin/online-gc-tester generate --base-dir=$BUILD_CACHE - - ./bin/online-gc-tester build --base-dir=$BUILD_CACHE - - ./bin/online-gc-tester push --base-dir=$BUILD_CACHE - - ./bin/online-gc-tester pull --base-dir=$BUILD_CACHE - - ./bin/online-gc-tester test --base-dir=$BUILD_CACHE --stage=1 --delay=$STAGE_ONE_VALIDATION_DELAY - - ./bin/online-gc-tester test --base-dir=$BUILD_CACHE --stage=2 --delay=$STAGE_TWO_VALIDATION_DELAY - - ./bin/online-gc-tester test --base-dir=$BUILD_CACHE --stage=3 --delay=$STAGE_THREE_VALIDATION_DELAY - - ./bin/online-gc-tester test --base-dir=$BUILD_CACHE --stage=4 --delay=$STAGE_FOUR_VALIDATION_DELAY - - ./bin/online-gc-tester test --base-dir=$BUILD_CACHE --stage=5 --delay=$STAGE_FIVE_VALIDATION_DELAY - timeout: 1h 30m - YAML - end - - before do - QA::Support::Helpers::ImportSource.enable('git') - - Flow::Login.sign_in - - imported_project - - Page::Project::Menu.perform(&:go_to_repository_settings) - - Page::Project::Settings::Repository.perform do |setting| - setting.expand_default_branch - end - - Page::Project::Settings::DefaultBranch.perform do |setting| - setting.set_default_branch('online-gc-test-builder-poc') - setting.click_save_changes_button - end - - Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do - Resource::Repository::Commit.fabricate_via_api! do |commit| - commit.project = imported_project - commit.branch = 'online-gc-test-builder-poc' - commit.commit_message = 'Update .gitlab-ci.yml' - commit.update_files([{ - file_path: '.gitlab-ci.yml', - content: gitlab_ci_yaml - }]) - end - end - end - - it 'runs the online garbage collector tool', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347663' do - imported_project.visit! - - Flow::Pipeline.visit_latest_pipeline - - Page::Project::Pipeline::Show.perform do |pipeline| - pipeline.click_job('test') - end - - Page::Project::Job::Show.perform do |job| - expect(job).to be_successful(timeout: 3900) - end - end - end - end -end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/container_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/container_registry_spec.rb new file mode 100644 index 00000000000..4b2d9f96cd2 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/container_registry_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package' do + describe 'SaaS Container Registry', only: { subdomain: %i[staging staging-canary pre] }, + product_group: :container_registry do + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-registry' + project.template_name = 'express' + end + end + + let(:registry_repository) do + Resource::RegistryRepository.fabricate! do |repository| + repository.name = project.path_with_namespace.to_s + repository.project = project + end + end + + let!(:gitlab_ci_yaml) do + <<~YAML + build: + image: docker:24.0.1 + stage: build + services: + - docker:24.0.1-dind + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + before_script: + - | + echo "Waiting for docker to start..." + for i in $(seq 1 30) + do + docker info && break + sleep 1s + done + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + YAML + end + + after do + registry_repository&.remove_via_api! + end + + it 'pushes project image to the container registry and deletes tag', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412806' do + Flow::Login.sign_in + project.visit! + + Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files([{ + file_path: '.gitlab-ci.yml', + content: gitlab_ci_yaml + }]) + end + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:go_to_container_registry) + + Page::Project::Registry::Show.perform do |registry| + expect(registry).to have_registry_repository(project.name) + + registry.click_on_image(project.name) + expect(registry).to have_tag('master') + + registry.click_delete + expect { registry.has_no_tag?('master') } + .to eventually_be_truthy.within(max_duration: 60, reload_page: page) + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb new file mode 100644 index 00000000000..9cf448aa8b7 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/self_managed/container_registry_spec.rb @@ -0,0 +1,267 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package', :orchestrated, :skip_live_env, product_group: :container_registry do + describe 'Self-managed Container Registry' do + include Support::Helpers::MaskToken + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-registry' + project.template_name = 'express' + project.visibility = :private + end + end + + let(:project_deploy_token) do + Resource::ProjectDeployToken.fabricate_via_api! do |deploy_token| + deploy_token.name = 'registry-deploy-token' + deploy_token.project = project + deploy_token.scopes = %w[ + read_repository + read_package_registry + write_package_registry + read_registry + write_registry + ] + end + end + + let(:registry_repository) do + Resource::RegistryRepository.fabricate! do |repository| + repository.name = project.path_with_namespace.to_s + repository.project = project + end + end + + let!(:runner) do + Resource::ProjectRunner.fabricate! do |runner| + runner.name = "qa-runner-#{Time.now.to_i}" + runner.tags = ["runner-for-#{project.name}"] + runner.executor = :docker + runner.project = project + end + end + + let(:personal_access_token) { Runtime::Env.personal_access_token } + + before do + Flow::Login.sign_in + project.visit! + end + + after do + registry_repository&.remove_via_api! + runner.remove_via_api! + end + + context "when tls is disabled" do + where do + { + 'using docker:20.10.23 and a personal access token' => { + docker_client_version: 'docker:20.10.23', + authentication_token_type: :personal_access_token, + token_name: 'Personal Access Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412807' + }, + 'using docker:20.10.23 and a project deploy token' => { + docker_client_version: 'docker:20.10.23', + authentication_token_type: :project_deploy_token, + token_name: 'Deploy Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412808' + }, + 'using docker:20.10.23 and a ci job token' => { + docker_client_version: 'docker:20.10.23', + authentication_token_type: :ci_job_token, + token_name: 'Job Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412809' + }, + 'using docker:23.0.6 and a personal access token' => { + docker_client_version: 'docker:23.0.6', + authentication_token_type: :personal_access_token, + token_name: 'Personal Access Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412810' + }, + 'using docker:23.0.6 and a project deploy token' => { + docker_client_version: 'docker:23.0.6', + authentication_token_type: :project_deploy_token, + token_name: 'Deploy Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412813' + }, + 'using docker:23.0.6 and a ci job token' => { + docker_client_version: 'docker:23.0.6', + authentication_token_type: :ci_job_token, + token_name: 'Job Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412814' + }, + 'using docker:24.0.1 and a personal access token' => { + docker_client_version: 'docker:24.0.1', + authentication_token_type: :personal_access_token, + token_name: 'Personal Access Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412817' + }, + 'using docker:24.0.1 and a project deploy token' => { + docker_client_version: 'docker:24.0.1', + authentication_token_type: :project_deploy_token, + token_name: 'Deploy Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412818' + }, + 'using docker:24.0.1 and a ci job token' => { + docker_client_version: 'docker:24.0.1', + authentication_token_type: :ci_job_token, + token_name: 'Job Token', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412819' + } + } + end + + with_them do + let(:auth_token) do + case authentication_token_type + when :personal_access_token + use_ci_variable(name: 'PERSONAL_ACCESS_TOKEN', value: personal_access_token, project: project) + when :project_deploy_token + use_ci_variable(name: 'PROJECT_DEPLOY_TOKEN', value: project_deploy_token.token, project: project) + when :ci_job_token + '$CI_JOB_TOKEN' + end + end + + let(:auth_user) do + case authentication_token_type + when :personal_access_token + "$CI_REGISTRY_USER" + when :project_deploy_token + "\"#{project_deploy_token.username}\"" + when :ci_job_token + 'gitlab-ci-token' + end + end + + it "pushes image and deletes tag", :registry, testcase: params[:testcase] do + Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + build: + image: "#{docker_client_version}" + stage: build + services: + - name: "#{docker_client_version}-dind" + command: ["--insecure-registry=gitlab.test:5050"] + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + script: + - docker login -u #{auth_user} -p #{auth_token} gitlab.test:5050 + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + tags: + - "runner-for-#{project.name}" + YAML + } + ] + ) + end + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + + Page::Project::Menu.perform(&:go_to_container_registry) + + Page::Project::Registry::Show.perform do |registry| + expect(registry).to have_registry_repository(project.name) + + registry.click_on_image(project.name) + expect(registry).to have_tag('master') + end + end + end + end + + context 'when tls is enabled' do + it( + 'pushes image and deletes tag', + :registry_tls, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347591' + ) do + Support::Retrier.retry_on_exception(max_attempts: 3, sleep_interval: 2) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: + <<~YAML + build: + image: docker:23.0.6 + stage: build + services: + - name: docker:23.0.6-dind + command: + - /bin/sh + - -c + - | + apk add --no-cache openssl + true | openssl s_client -showcerts -connect gitlab.test:5050 > /usr/local/share/ca-certificates/gitlab.test.crt + update-ca-certificates + dockerd-entrypoint.sh || exit + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD gitlab.test:5050 + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + tags: + - "runner-for-#{project.name}" + YAML + } + ] + ) + end + end + + Flow::Pipeline.visit_latest_pipeline + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('build') + end + + Support::Retrier.retry_until(max_duration: 800, sleep_interval: 10) do + project.pipelines.last[:status] == 'success' + end + + Page::Project::Menu.perform(&:go_to_container_registry) + + Page::Project::Registry::Show.perform do |registry| + expect(registry).to have_registry_repository(project.name) + + registry.click_on_image(project.name) + + expect(registry).to have_tag('master') + + registry.click_delete + + expect { registry.has_no_tag?('master') } + .to eventually_be_truthy.within(max_duration: 60, reload_page: page) + end + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb index 7e99cdba369..026fd3c5d10 100644 --- a/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/dependency_proxy/dependency_proxy_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :orchestrated, :registry, only: { pipeline: :main }, product_group: :container_registry do + RSpec.describe 'Package', :orchestrated, :registry, :skip_live_env, product_group: :container_registry do describe 'Dependency Proxy' do using RSpec::Parameterized::TableSyntax include Support::Helpers::MaskToken @@ -59,59 +59,23 @@ module QA where do { - 'using docker:18.09.9 and a personal access token' => { - docker_client_version: 'docker:18.09.9', + 'using docker:24.0.1 and a personal access token' => { + docker_client_version: 'docker:24.0.1', authentication_token_type: :personal_access_token, token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370195' + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412820' }, - 'using docker:18.09.9 and a group deploy token' => { - docker_client_version: 'docker:18.09.9', + 'using docker:24.0.1 and a group deploy token' => { + docker_client_version: 'docker:24.0.1', authentication_token_type: :group_deploy_token, token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370196' + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412821' }, - 'using docker:18.09.9 and a ci job token' => { - docker_client_version: 'docker:18.09.9', + 'using docker:24.0.1 and a ci job token' => { + docker_client_version: 'docker:24.0.1', authentication_token_type: :ci_job_token, token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370198' - }, - 'using docker:19.03.12 and a personal access token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :personal_access_token, - token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370189' - }, - 'using docker:19.03.12 and a group deploy token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :group_deploy_token, - token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370190' - }, - 'using docker:19.03.12 and a ci job token' => { - docker_client_version: 'docker:19.03.12', - authentication_token_type: :ci_job_token, - token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370191' - }, - 'using docker:20.10 and a personal access token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :personal_access_token, - token_name: 'Personal Access Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370192' - }, - 'using docker:20.10 and a group deploy token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :group_deploy_token, - token_name: 'Deploy Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370193' - }, - 'using docker:20.10 and a ci job token' => { - docker_client_version: 'docker:20.10', - authentication_token_type: :ci_job_token, - token_name: 'Job Token', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/370194' + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412822' } } end diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 53674e9cb90..536da48f07f 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -53,6 +53,13 @@ sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml cp config/resque.yml.example config/resque.yml sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml +# By default, run CI against Redis Cluster to ensure Redis Cluster compatibility. +# if SETUP_DB is false, the jobs are not backend-related +if [[ "$USE_REDIS_CLUSTER" != "false" ]] && [[ "$SETUP_DB" != "false" ]]; then + cp config/redis.yml.example config/redis.yml + sed -i 's|- .*$|- redis://rediscluster:7001|g' config/redis.yml +fi + if [ "$SETUP_DB" != "false" ]; then setup_db elif getent hosts postgres; then diff --git a/spec/controllers/admin/dashboard_controller_spec.rb b/spec/controllers/admin/dashboard_controller_spec.rb index bfbd2ca946f..e3d579a631c 100644 --- a/spec/controllers/admin/dashboard_controller_spec.rb +++ b/spec/controllers/admin/dashboard_controller_spec.rb @@ -11,7 +11,8 @@ RSpec.describe Admin::DashboardController do it 'retrieves Redis versions' do get :index - expect(assigns[:redis_versions].length).to eq(1) + # specs are run against both Redis and Redis Cluster instances. + expect(assigns[:redis_versions].length).to be > 0 end context 'with pending_delete projects' do diff --git a/spec/lib/bulk_imports/clients/http_spec.rb b/spec/lib/bulk_imports/clients/http_spec.rb index aff049408e2..bf1bfb77b26 100644 --- a/spec/lib/bulk_imports/clients/http_spec.rb +++ b/spec/lib/bulk_imports/clients/http_spec.rb @@ -33,11 +33,17 @@ RSpec.describe BulkImports::Clients::HTTP, feature_category: :importers do end context 'error handling' do - context 'when error occurred' do - it 'raises BulkImports::NetworkError' do - allow(Gitlab::HTTP).to receive(method).and_raise(Errno::ECONNREFUSED) + context 'when any known HTTP error occurs' do + using RSpec::Parameterized::TableSyntax + + where(:exception_class) { Gitlab::HTTP::HTTP_ERRORS } + + with_them do + it 'raises BulkImports::NetworkError' do + allow(Gitlab::HTTP).to receive(method).and_raise(exception_class) - expect { subject.public_send(method, resource) }.to raise_exception(BulkImports::NetworkError) + expect { subject.public_send(method, resource) }.to raise_exception(BulkImports::NetworkError) + end end end diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index ee0854c2483..01875aadacb 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -161,7 +161,7 @@ - './ee/spec/controllers/projects/subscriptions_controller_spec.rb' - './ee/spec/controllers/projects/vulnerability_feedback_controller_spec.rb' - './ee/spec/controllers/registrations/company_controller_spec.rb' -- './ee/spec/controllers/registrations/groups_projects_controller_spec.rb' +- './ee/spec/controllers/registrations/groups_controller_spec.rb' - './ee/spec/controllers/repositories/git_http_controller_spec.rb' - './ee/spec/controllers/security/dashboard_controller_spec.rb' - './ee/spec/controllers/security/projects_controller_spec.rb' @@ -3079,7 +3079,7 @@ - './ee/spec/views/projects/security/policies/index.html.haml_spec.rb' - './ee/spec/views/projects/security/sast_configuration/show.html.haml_spec.rb' - './ee/spec/views/projects/settings/subscriptions/_index.html.haml_spec.rb' -- './ee/spec/views/registrations/groups_projects/new.html.haml_spec.rb' +- './ee/spec/views/registrations/groups/new.html.haml_spec.rb' - './ee/spec/views/shared/billings/_billing_plan_actions.html.haml_spec.rb' - './ee/spec/views/shared/billings/_billing_plan.html.haml_spec.rb' - './ee/spec/views/shared/billings/_billing_plans.html.haml_spec.rb' -- cgit v1.2.3