diff options
43 files changed, 235 insertions, 399 deletions
diff --git a/app/assets/stylesheets/framework/sidebar.scss b/app/assets/stylesheets/framework/sidebar.scss index 7c5ccd07913..dd9581c4692 100644 --- a/app/assets/stylesheets/framework/sidebar.scss +++ b/app/assets/stylesheets/framework/sidebar.scss @@ -93,7 +93,7 @@ } .right-sidebar { - border-left: 1px solid $gray-50; + border-left: 1px solid $gray-100; &.right-sidebar-merge-requests { @include media-breakpoint-up(md) { diff --git a/config/feature_flags/development/ci_authenticate_running_job_token_for_artifacts.yml b/config/feature_flags/development/ci_authenticate_running_job_token_for_artifacts.yml deleted file mode 100644 index 653380f445e..00000000000 --- a/config/feature_flags/development/ci_authenticate_running_job_token_for_artifacts.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_authenticate_running_job_token_for_artifacts -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83713 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357075 -milestone: '15.0' -type: development -group: group::pipeline insights -default_enabled: false diff --git a/config/feature_flags/development/ci_expose_running_job_token_for_artifacts.yml b/config/feature_flags/development/ci_expose_running_job_token_for_artifacts.yml deleted file mode 100644 index aa6bcbeb1d1..00000000000 --- a/config/feature_flags/development/ci_expose_running_job_token_for_artifacts.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_expose_running_job_token_for_artifacts -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83713 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357075 -milestone: '15.0' -type: development -group: group::pipeline insights -default_enabled: false diff --git a/data/removals/15_0/15-0-praefect-virtual-storage.yml b/data/removals/15_0/15-0-praefect-virtual-storage.yml new file mode 100644 index 00000000000..a1ffa20bcfd --- /dev/null +++ b/data/removals/15_0/15-0-praefect-virtual-storage.yml @@ -0,0 +1,9 @@ +- name: "Gitaly nodes in virtual storage" + announcement_milestone: "13.12" # The milestone when this feature was first announced as deprecated. + announcement_date: "2021-04-22" + removal_milestone: "15.0" # the milestone when this feature is planned to be removed + removal_date: "2022-05-22" # the date of the milestone release when this feature is planned to be removed + breaking_change: true + body: | # Do not modify this line, instead modify the lines below. + Configuring the Gitaly nodes directly in the virtual storage's root configuration object has been deprecated in GitLab 13.12 and is no longer supported in GitLab 15.0. You must move the Gitaly nodes under the `'nodes'` key as described in [the Praefect configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#praefect). + tiers: [Free, Premium, Ultimate] diff --git a/db/docs/ci_job_artifact_states.yml b/db/docs/ci_job_artifact_states.yml index d555c2d4564..6fe7443dd26 100644 --- a/db/docs/ci_job_artifact_states.yml +++ b/db/docs/ci_job_artifact_states.yml @@ -4,6 +4,6 @@ classes: - Geo::JobArtifactState feature_categories: - geo_replication -description: TODO +description: Separate table for job artifacts containing Geo verification metadata. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75264 milestone: '14.8' diff --git a/db/docs/geo_cache_invalidation_events.yml b/db/docs/geo_cache_invalidation_events.yml index 6ae56893281..5695cddfb7f 100644 --- a/db/docs/geo_cache_invalidation_events.yml +++ b/db/docs/geo_cache_invalidation_events.yml @@ -4,6 +4,6 @@ classes: - Geo::CacheInvalidationEvent feature_categories: - geo_replication -description: TODO +description: Geo event to process feature flag toggles instantly on a secondary by invalidating the cache, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7738 milestone: '11.4' diff --git a/db/docs/geo_container_repository_updated_events.yml b/db/docs/geo_container_repository_updated_events.yml index c48c60fbf10..8d17ded97d4 100644 --- a/db/docs/geo_container_repository_updated_events.yml +++ b/db/docs/geo_container_repository_updated_events.yml @@ -4,6 +4,6 @@ classes: - Geo::ContainerRepositoryUpdatedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a container repository (image, tag, registry) gets updated, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/1902d9cc74a1dc2c87fdbb39a6cdbb67092cbb5a milestone: '12.2' diff --git a/db/docs/geo_event_log.yml b/db/docs/geo_event_log.yml index 122ded0537e..905383664e5 100644 --- a/db/docs/geo_event_log.yml +++ b/db/docs/geo_event_log.yml @@ -4,6 +4,6 @@ classes: - Geo::EventLog feature_categories: - geo_replication -description: TODO +description: Log of all events that a Geo secondary can process. Parsed/watched through streaming replication on all secondaries. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/cb6c7cbe2a9ee05cea6926e3d8c18f6aa26f4c64 milestone: '9.3' diff --git a/db/docs/geo_events.yml b/db/docs/geo_events.yml index 8a56c02d33e..6e30da25f5a 100644 --- a/db/docs/geo_events.yml +++ b/db/docs/geo_events.yml @@ -4,6 +4,6 @@ classes: - Geo::Event feature_categories: - geo_replication -description: TODO +description: Geo events implemented generically, used by the SSF where all object types can generate an event to be processed by the secondary sites. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23447 milestone: '12.8' diff --git a/db/docs/geo_hashed_storage_attachments_events.yml b/db/docs/geo_hashed_storage_attachments_events.yml index 6d98ea7d21b..457b34af0f1 100644 --- a/db/docs/geo_hashed_storage_attachments_events.yml +++ b/db/docs/geo_hashed_storage_attachments_events.yml @@ -4,6 +4,6 @@ classes: - Geo::HashedStorageAttachmentsEvent feature_categories: - geo_replication -description: TODO +description: Used to replicate storage attachments migration paths on Geo secondaries from regular to hashed storage. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3544 milestone: '10.3' diff --git a/db/docs/geo_hashed_storage_migrated_events.yml b/db/docs/geo_hashed_storage_migrated_events.yml index 38415c32f52..175015c487d 100644 --- a/db/docs/geo_hashed_storage_migrated_events.yml +++ b/db/docs/geo_hashed_storage_migrated_events.yml @@ -4,6 +4,6 @@ classes: - Geo::HashedStorageMigratedEvent feature_categories: - geo_replication -description: TODO +description: Used to replicate repository migration paths on Geo secondaries from regular to hashed storage. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3066 milestone: '10.2' diff --git a/db/docs/geo_lfs_object_deleted_events.yml b/db/docs/geo_lfs_object_deleted_events.yml index 1bf82510ee4..05cafd6d806 100644 --- a/db/docs/geo_lfs_object_deleted_events.yml +++ b/db/docs/geo_lfs_object_deleted_events.yml @@ -3,6 +3,6 @@ table_name: geo_lfs_object_deleted_events classes: [] feature_categories: - geo_replication -description: TODO +description: Geo event for when an LFS object gets deleted, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3481 milestone: '10.2' diff --git a/db/docs/geo_node_namespace_links.yml b/db/docs/geo_node_namespace_links.yml index 6847760f494..cede321ed47 100644 --- a/db/docs/geo_node_namespace_links.yml +++ b/db/docs/geo_node_namespace_links.yml @@ -4,6 +4,6 @@ classes: - GeoNodeNamespaceLink feature_categories: - geo_replication -description: TODO +description: Passthrough table for geo_nodes many-to-many namespaces relation. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/14c6128530579ca92fa79342d4119d25bcff1f2d milestone: '9.5' diff --git a/db/docs/geo_node_statuses.yml b/db/docs/geo_node_statuses.yml index 3a7dfe16df1..ff955e79eb1 100644 --- a/db/docs/geo_node_statuses.yml +++ b/db/docs/geo_node_statuses.yml @@ -4,6 +4,6 @@ classes: - GeoNodeStatus feature_categories: - geo_replication -description: TODO +description: Contains sites status and metadata for each Geo site, updated async through a scheduled worker. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3230 milestone: '10.2' diff --git a/db/docs/geo_nodes.yml b/db/docs/geo_nodes.yml index 143213cad96..956b79fe0a0 100644 --- a/db/docs/geo_nodes.yml +++ b/db/docs/geo_nodes.yml @@ -4,6 +4,6 @@ classes: - GeoNode feature_categories: - geo_replication -description: TODO +description: Contains Geo sites configuration data and settings. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/5ab12ad02ed753dd933485094ba45512890f0b50 milestone: '8.5' diff --git a/db/docs/geo_repositories_changed_events.yml b/db/docs/geo_repositories_changed_events.yml index 214a4b3fe09..348d8331fe7 100644 --- a/db/docs/geo_repositories_changed_events.yml +++ b/db/docs/geo_repositories_changed_events.yml @@ -4,6 +4,6 @@ classes: - Geo::RepositoriesChangedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when the repositories for selective sync of a specific Geo secondary change, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/312bc703a4619b87ba2ac4e59623e7747a24502c milestone: '9.5' diff --git a/db/docs/geo_repository_created_events.yml b/db/docs/geo_repository_created_events.yml index 22c546ac0a2..ea7b8558bdc 100644 --- a/db/docs/geo_repository_created_events.yml +++ b/db/docs/geo_repository_created_events.yml @@ -4,6 +4,6 @@ classes: - Geo::RepositoryCreatedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a repository gets created, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/f3eacf881659b7af97b7c7ba3289237ec6cdc1cb milestone: '10.0' diff --git a/db/docs/geo_repository_deleted_events.yml b/db/docs/geo_repository_deleted_events.yml index 9c2b72a0172..4814994181a 100644 --- a/db/docs/geo_repository_deleted_events.yml +++ b/db/docs/geo_repository_deleted_events.yml @@ -4,6 +4,6 @@ classes: - Geo::RepositoryDeletedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a repository gets deleted, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/04c3da24ac5975b140cf2e6a7e33414543f148f5 milestone: '9.4' diff --git a/db/docs/geo_repository_renamed_events.yml b/db/docs/geo_repository_renamed_events.yml index 431577572e3..2e6838f51e4 100644 --- a/db/docs/geo_repository_renamed_events.yml +++ b/db/docs/geo_repository_renamed_events.yml @@ -4,6 +4,6 @@ classes: - Geo::RepositoryRenamedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a repository gets renamed, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/6e5fa040d1c689fad4e110dd10be8ddba61ea7ef milestone: '9.4' diff --git a/db/docs/geo_repository_updated_events.yml b/db/docs/geo_repository_updated_events.yml index 0aa83949fef..51a0033d0a7 100644 --- a/db/docs/geo_repository_updated_events.yml +++ b/db/docs/geo_repository_updated_events.yml @@ -4,6 +4,6 @@ classes: - Geo::RepositoryUpdatedEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a repository gets updated (content changed), belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/71cc57b1e4b7721c93107357517235a18f7ba8e2 milestone: '9.3' diff --git a/db/docs/geo_reset_checksum_events.yml b/db/docs/geo_reset_checksum_events.yml index ba56f659295..d4aad156687 100644 --- a/db/docs/geo_reset_checksum_events.yml +++ b/db/docs/geo_reset_checksum_events.yml @@ -4,6 +4,6 @@ classes: - Geo::ResetChecksumEvent feature_categories: - geo_replication -description: TODO +description: Geo event for when a project gets reverified on the primary, belongs to geo_event_log. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7394 milestone: '11.4' diff --git a/db/docs/upload_states.yml b/db/docs/upload_states.yml index b1ee7e17e2e..29df41c07dd 100644 --- a/db/docs/upload_states.yml +++ b/db/docs/upload_states.yml @@ -4,6 +4,6 @@ classes: - Geo::UploadState feature_categories: - geo_replication -description: TODO +description: Separate table for uploads containing Geo verification metadata. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65921 milestone: '14.6' diff --git a/doc/install/docker.md b/doc/install/docker.md index 61e85778d97..d5ca2c2e29f 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -189,7 +189,7 @@ services: external_url 'http://gitlab.example.com:8929' gitlab_rails['gitlab_shell_ssh_port'] = 2224 ports: - - '8929:80' + - '8929:8929' - '2224:22' volumes: - '$GITLAB_HOME/config:/etc/gitlab' @@ -198,7 +198,7 @@ services: shm_size: '256m' ``` -This is the same as using `--publish 8929:80 --publish 2224:22`. +This is the same as using `--publish 8929:8929 --publish 2224:22`. ### Install GitLab using Docker swarm mode diff --git a/doc/update/removals.md b/doc/update/removals.md index c6924388999..4e653d5ab0a 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -224,6 +224,16 @@ All functionality related to GitLab Serverless was deprecated in GitLab 14.3 and For additional context, or to provide feedback regarding this change, please reference our [deprecation issue](https://gitlab.com/groups/gitlab-org/configure/-/epics/6). +### Gitaly nodes in virtual storage + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Configuring the Gitaly nodes directly in the virtual storage's root configuration object has been deprecated in GitLab 13.12 and is no longer supported in GitLab 15.0. You must move the Gitaly nodes under the `'nodes'` key as described in [the Praefect configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#praefect). + ### GraphQL permissions change for Package settings WARNING: diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index b4e411b0457..4381309fb9e 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -324,8 +324,7 @@ module API optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end get '/:id/artifacts', feature_category: :build_artifacts do - if Feature.enabled?(:ci_authenticate_running_job_token_for_artifacts, current_job&.project) && - request_using_running_job_token? + if request_using_running_job_token? authenticate_job_via_dependent_job! else authenticate_job!(require_running: false) diff --git a/lib/api/entities/ci/job_request/dependency.rb b/lib/api/entities/ci/job_request/dependency.rb index 791bfba1079..63c1552de8a 100644 --- a/lib/api/entities/ci/job_request/dependency.rb +++ b/lib/api/entities/ci/job_request/dependency.rb @@ -8,11 +8,7 @@ module API expose :id, :name expose :token do |job, options| - if ::Feature.enabled?(:ci_expose_running_job_token_for_artifacts, job.project) - options[:running_job]&.token - else - job.token - end + options[:running_job]&.token end expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.available_artifacts? } diff --git a/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb index d6c6b5d44a8..d4f5901e971 100644 --- a/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb @@ -8,7 +8,7 @@ module BulkImports file_extraction_pipeline! - relation_name 'self' + relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index a187a4821b9..1754f27137c 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -8,6 +8,8 @@ module BulkImports file_extraction_pipeline! + relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer def extract(_context) @@ -57,7 +59,7 @@ module BulkImports def download_service @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + relative_url: context.entity.relation_download_url_path(self.class.relation), tmpdir: tmpdir, filename: compressed_filename ) @@ -72,7 +74,7 @@ module BulkImports end def filename - "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json" + "#{self.class.relation}.json" end def json_decode(string) diff --git a/spec/lib/api/entities/ci/job_request/dependency_spec.rb b/spec/lib/api/entities/ci/job_request/dependency_spec.rb index b45885e9982..bbeb864c2ee 100644 --- a/spec/lib/api/entities/ci/job_request/dependency_spec.rb +++ b/spec/lib/api/entities/ci/job_request/dependency_spec.rb @@ -21,16 +21,6 @@ RSpec.describe API::Entities::Ci::JobRequest::Dependency do expect(subject[:token]).to eq(running_job.token) end - context 'when ci_expose_running_job_token_for_artifacts is disabled' do - before do - stub_feature_flags(ci_expose_running_job_token_for_artifacts: false) - end - - it 'returns the token belonging to the dependency job' do - expect(subject[:token]).to eq(job.token) - end - end - it 'returns the dependency artifacts_file', :aggregate_failures do expect(subject[:artifacts_file][:filename]).to eq('ci_build_artifacts.zip') expect(subject[:artifacts_file][:size]).to eq(job.artifacts_size) diff --git a/spec/lib/bulk_imports/groups/pipelines/group_attributes_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/group_attributes_pipeline_spec.rb index 29ac9c916d9..7ac417afa0b 100644 --- a/spec/lib/bulk_imports/groups/pipelines/group_attributes_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/group_attributes_pipeline_spec.rb @@ -73,4 +73,8 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupAttributesPipeline do pipeline.after_run(nil) end end + + describe '.relation' do + it { expect(described_class.relation).to eq('self') } + end end diff --git a/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb index 6e1fd252c25..aa9c7486c27 100644 --- a/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb +++ b/spec/lib/bulk_imports/projects/pipelines/project_attributes_pipeline_spec.rb @@ -176,4 +176,8 @@ RSpec.describe BulkImports::Projects::Pipelines::ProjectAttributesPipeline do end end end + + describe '.relation' do + it { expect(described_class.relation).to eq('self') } + end end diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index c74bd933b22..4a1a9beb21a 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -49,9 +49,10 @@ RSpec.describe Gitlab::SidekiqConfig do before do allow(described_class).to receive(:workers).and_return(workers) allow(Gitlab).to receive(:ee?).and_return(false) + allow(Gitlab).to receive(:jh?).and_return(false) end - it 'returns true if the YAML file does not matcph the application code' do + it 'returns true if the YAML file does not match the application code' do allow(YAML).to receive(:load_file) .with(described_class::FOSS_QUEUE_CONFIG_PATH) .and_return(workers.first(2).map(&:to_yaml)) @@ -101,6 +102,7 @@ RSpec.describe Gitlab::SidekiqConfig do ].map { |worker| described_class::Worker.new(worker, ee: false) } allow(described_class).to receive(:workers).and_return(workers) + allow(Gitlab).to receive(:jh?).and_return(false) end let(:expected_queues) do diff --git a/spec/requests/api/ci/runner/jobs_artifacts_spec.rb b/spec/requests/api/ci/runner/jobs_artifacts_spec.rb index 8fabd37c854..5767fa4326e 100644 --- a/spec/requests/api/ci/runner/jobs_artifacts_spec.rb +++ b/spec/requests/api/ci/runner/jobs_artifacts_spec.rb @@ -895,22 +895,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do it_behaves_like 'successful artifact download' end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(ci_authenticate_running_job_token_for_artifacts: false) - end - - it_behaves_like 'successful artifact download' - - context 'when the job is no longer running' do - before do - job.success! - end - - it_behaves_like 'successful artifact download' - end - end end context 'when using token belonging to the dependent job' do @@ -928,14 +912,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do it_behaves_like 'forbidden request' end - - context 'when feature flag is disabled' do - before do - stub_feature_flags(ci_authenticate_running_job_token_for_artifacts: false) - end - - it_behaves_like 'forbidden request' - end end context 'when using token belonging to another job created by another project member' do @@ -952,14 +928,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do end it_behaves_like 'successful artifact download' - - context 'when feature flag is disabled' do - before do - stub_feature_flags(ci_authenticate_running_job_token_for_artifacts: false) - end - - it_behaves_like 'forbidden request' - end end context 'when using token belonging to a pending dependent job' do diff --git a/spec/requests/api/ci/runner/jobs_request_post_spec.rb b/spec/requests/api/ci/runner/jobs_request_post_spec.rb index b54ad811e6e..dbc5f0e74e2 100644 --- a/spec/requests/api/ci/runner/jobs_request_post_spec.rb +++ b/spec/requests/api/ci/runner/jobs_request_post_spec.rb @@ -507,23 +507,6 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do { 'id' => job2.id, 'name' => job2.name, 'token' => test_job.token }) end - context 'when ci_expose_running_job_token_for_artifacts is disabled' do - before do - stub_feature_flags(ci_expose_running_job_token_for_artifacts: false) - end - - it 'returns dependent jobs with the dependency job tokens' do - request_job - - expect(response).to have_gitlab_http_status(:created) - expect(json_response['id']).to eq(test_job.id) - expect(json_response['dependencies'].count).to eq(2) - expect(json_response['dependencies']).to include( - { 'id' => job.id, 'name' => job.name, 'token' => job.token }, - { 'id' => job2.id, 'name' => job2.name, 'token' => job2.token }) - end - end - describe 'preloading job_artifacts_archive' do it 'queries the ci_job_artifacts table once only' do expect { request_job }.not_to exceed_all_query_limit(1).for_model(::Ci::JobArtifact) diff --git a/spec/support/gitlab/usage/metrics_instrumentation_shared_examples.rb b/spec/support/gitlab/usage/metrics_instrumentation_shared_examples.rb index 4624a8ac82a..e02bf66507a 100644 --- a/spec/support/gitlab/usage/metrics_instrumentation_shared_examples.rb +++ b/spec/support/gitlab/usage/metrics_instrumentation_shared_examples.rb @@ -5,6 +5,10 @@ RSpec.shared_examples 'a correct instrumented metric value' do |params| let(:options) { params[:options] } let(:metric) { described_class.new(time_frame: time_frame, options: options) } + around do |example| + freeze_time { example.run } + end + before do if described_class.respond_to?(:relation) && described_class.relation.respond_to?(:connection) allow(described_class.relation.connection).to receive(:transaction_open?).and_return(false) diff --git a/spec/views/devise/shared/_signup_box.html.haml_spec.rb b/spec/views/devise/shared/_signup_box.html.haml_spec.rb index 1f0cd213f7b..b0730e6fc54 100644 --- a/spec/views/devise/shared/_signup_box.html.haml_spec.rb +++ b/spec/views/devise/shared/_signup_box.html.haml_spec.rb @@ -3,28 +3,41 @@ require 'spec_helper' RSpec.describe 'devise/shared/_signup_box' do + let(:button_text) { '_button_text_' } + let(:terms_path) { '_terms_path_' } + + let(:translation_com) do + s_("SignUp|By clicking %{button_text}, I agree that I have read and accepted "\ + "the GitLab %{link_start}Terms of Use and Privacy Policy%{link_end}") + end + + let(:translation_non_com) do + s_("SignUp|By clicking %{button_text}, I agree that I have read and accepted "\ + "the %{link_start}Terms of Use and Privacy Policy%{link_end}") + end + before do stub_devise allow(view).to receive(:show_omniauth_providers).and_return(false) allow(view).to receive(:url).and_return('_url_') - allow(view).to receive(:terms_path).and_return('_terms_path_') - allow(view).to receive(:button_text).and_return('_button_text_') + allow(view).to receive(:terms_path).and_return(terms_path) + allow(view).to receive(:button_text).and_return(button_text) allow(view).to receive(:signup_username_data_attributes).and_return({}) stub_template 'devise/shared/_error_messages.html.haml' => '' end + def text(translation) + format(translation, + button_text: button_text, + link_start: "<a href='#{terms_path}' target='_blank' rel='noreferrer noopener'>", + link_end: '</a>') + end + context 'when terms are enforced' do before do allow(Gitlab::CurrentSettings.current_application_settings).to receive(:enforce_terms?).and_return(true) end - it 'shows expected text with placeholders' do - render - - expect(rendered).to have_content('By clicking _button_text_') - expect(rendered).to have_link('Terms of Use and Privacy Policy') - end - context 'when on .com' do before do allow(Gitlab).to receive(:com?).and_return(true) @@ -33,7 +46,7 @@ RSpec.describe 'devise/shared/_signup_box' do it 'shows expected GitLab text' do render - expect(rendered).to have_content('I have read and accepted the GitLab Terms') + expect(rendered).to include(text(translation_com)) end end @@ -45,7 +58,7 @@ RSpec.describe 'devise/shared/_signup_box' do it 'shows expected text without GitLab' do render - expect(rendered).to have_content('I have read and accepted the Terms') + expect(rendered).to include(text(translation_non_com)) end end end @@ -59,7 +72,7 @@ RSpec.describe 'devise/shared/_signup_box' do it 'shows expected text with placeholders' do render - expect(rendered).not_to have_content('By clicking') + expect(rendered).not_to include(text(translation_com)) end end diff --git a/workhorse/internal/server/server.go b/workhorse/internal/server/server.go deleted file mode 100644 index e6ad5bff8d3..00000000000 --- a/workhorse/internal/server/server.go +++ /dev/null @@ -1,107 +0,0 @@ -package server - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "syscall" - - "gitlab.com/gitlab-org/labkit/log" - - "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" -) - -var tlsVersions = map[string]uint16{ - "": 0, // Default value in tls.Config - "tls1.0": tls.VersionTLS10, - "tls1.1": tls.VersionTLS11, - "tls1.2": tls.VersionTLS12, - "tls1.3": tls.VersionTLS13, -} - -type Server struct { - Handler http.Handler - Umask int - ListenerConfigs []config.ListenerConfig - Errors chan error - - servers []*http.Server -} - -func (s *Server) Run() error { - oldUmask := syscall.Umask(s.Umask) - defer syscall.Umask(oldUmask) - - for _, cfg := range s.ListenerConfigs { - listener, err := s.newListener("upstream", cfg) - if err != nil { - return fmt.Errorf("server.Run: failed creating a listener: %v", err) - } - - s.runUpstreamServer(listener) - } - - return nil -} - -func (s *Server) Close() error { - return s.allServers(func(srv *http.Server) error { return srv.Close() }) -} - -func (s *Server) Shutdown(ctx context.Context) error { - return s.allServers(func(srv *http.Server) error { return srv.Shutdown(ctx) }) -} - -func (s *Server) allServers(callback func(*http.Server) error) error { - var resultErr error - errC := make(chan error, len(s.servers)) - for _, server := range s.servers { - server := server // Capture loop variable - go func() { errC <- callback(server) }() - } - - for range s.servers { - if err := <-errC; err != nil { - resultErr = err - } - } - - return resultErr -} - -func (s *Server) runUpstreamServer(listener net.Listener) { - srv := &http.Server{ - Addr: listener.Addr().String(), - Handler: s.Handler, - } - go func() { - s.Errors <- srv.Serve(listener) - }() - - s.servers = append(s.servers, srv) -} - -func (s *Server) newListener(name string, cfg config.ListenerConfig) (net.Listener, error) { - if cfg.Tls == nil { - log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server", name) - - return net.Listen(cfg.Network, cfg.Addr) - } - - cert, err := tls.LoadX509KeyPair(cfg.Tls.Certificate, cfg.Tls.Key) - if err != nil { - return nil, err - } - - log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server with tls", name) - - tlsConfig := &tls.Config{ - MinVersion: tlsVersions[cfg.Tls.MinVersion], - MaxVersion: tlsVersions[cfg.Tls.MaxVersion], - Certificates: []tls.Certificate{cert}, - } - - return tls.Listen(cfg.Network, cfg.Addr, tlsConfig) -} diff --git a/workhorse/internal/server/server_test.go b/workhorse/internal/server/server_test.go deleted file mode 100644 index c342b4e96a7..00000000000 --- a/workhorse/internal/server/server_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package server - -import ( - "context" - "crypto/tls" - "crypto/x509" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" -) - -const ( - certFile = "testdata/localhost.crt" - keyFile = "testdata/localhost.key" -) - -func TestRun(t *testing.T) { - srv := defaultServer() - - require.NoError(t, srv.Run()) - defer srv.Close() - - require.Len(t, srv.servers, 2) - - clients := buildClients(t, srv.servers) - for url, client := range clients { - resp, err := client.Get(url) - require.NoError(t, err) - require.Equal(t, 200, resp.StatusCode) - } -} - -func TestShutdown(t *testing.T) { - ready := make(chan bool) - done := make(chan bool) - statusCodes := make(chan int) - - srv := defaultServer() - srv.Handler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - ready <- true - <-done - rw.WriteHeader(200) - }) - - require.NoError(t, srv.Run()) - defer srv.Close() - - clients := buildClients(t, srv.servers) - - for url, client := range clients { - go func(url string, client *http.Client) { - resp, err := client.Get(url) - require.NoError(t, err) - statusCodes <- resp.StatusCode - }(url, client) - } - - for range clients { - <-ready - } // initiate requests - - shutdownError := make(chan error) - go func() { - shutdownError <- srv.Shutdown(context.Background()) - }() - - for url, client := range clients { - require.Eventually(t, func() bool { - _, err := client.Get(url) - return err != nil - }, time.Second, 10*time.Millisecond, "server must stop accepting new requests") - } - - for range clients { - done <- true - } // finish requests - - require.NoError(t, <-shutdownError) - require.ElementsMatch(t, []int{200, 200}, []int{<-statusCodes, <-statusCodes}) -} - -func TestShutdown_withTimeout(t *testing.T) { - ready := make(chan bool) - done := make(chan bool) - - srv := defaultServer() - srv.Handler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - ready <- true - <-done - rw.WriteHeader(200) - }) - - require.NoError(t, srv.Run()) - defer srv.Close() - - clients := buildClients(t, srv.servers) - - for url, client := range clients { - go func(url string, client *http.Client) { - client.Get(url) - }(url, client) - } - - for range clients { - <-ready - } // initiate requets - - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - - err := srv.Shutdown(ctx) - require.Error(t, err) - require.EqualError(t, err, "context deadline exceeded") -} - -func defaultServer() Server { - return Server{ - ListenerConfigs: []config.ListenerConfig{ - { - Addr: "127.0.0.1:0", - Network: "tcp", - }, - { - Addr: "127.0.0.1:0", - Network: "tcp", - Tls: &config.TlsConfig{ - Certificate: certFile, - Key: keyFile, - }, - }, - }, - Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - rw.WriteHeader(200) - }), - Errors: make(chan error), - } -} - -func buildClients(t *testing.T, servers []*http.Server) map[string]*http.Client { - httpsClient := &http.Client{} - certpool := x509.NewCertPool() - - tlsCertificate, err := tls.LoadX509KeyPair(certFile, keyFile) - require.NoError(t, err) - - certificate, err := x509.ParseCertificate(tlsCertificate.Certificate[0]) - require.NoError(t, err) - - certpool.AddCert(certificate) - httpsClient.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certpool, - }, - } - - httpServer, httpsServer := servers[0], servers[1] - return map[string]*http.Client{ - "http://" + httpServer.Addr: http.DefaultClient, - "https://" + httpsServer.Addr: httpsClient, - } -} diff --git a/workhorse/listener.go b/workhorse/listener.go new file mode 100644 index 00000000000..5c8ca3a4064 --- /dev/null +++ b/workhorse/listener.go @@ -0,0 +1,41 @@ +package main + +import ( + "crypto/tls" + "net" + + "gitlab.com/gitlab-org/labkit/log" + + "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" +) + +var tlsVersions = map[string]uint16{ + "": 0, // Default value in tls.Config + "tls1.0": tls.VersionTLS10, + "tls1.1": tls.VersionTLS11, + "tls1.2": tls.VersionTLS12, + "tls1.3": tls.VersionTLS13, +} + +func newListener(name string, cfg config.ListenerConfig) (net.Listener, error) { + if cfg.Tls == nil { + log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server", name) + + return net.Listen(cfg.Network, cfg.Addr) + } + + cert, err := tls.LoadX509KeyPair(cfg.Tls.Certificate, cfg.Tls.Key) + if err != nil { + return nil, err + } + + log.WithFields(log.Fields{"address": cfg.Addr, "network": cfg.Network}).Infof("Running %v server with tls", name) + + tlsConfig := &tls.Config{ + MinVersion: tlsVersions[cfg.Tls.MinVersion], + MaxVersion: tlsVersions[cfg.Tls.MaxVersion], + Certificates: []tls.Certificate{cert}, + } + + return tls.Listen(cfg.Network, cfg.Addr, tlsConfig) +} diff --git a/workhorse/listener_test.go b/workhorse/listener_test.go new file mode 100644 index 00000000000..26b5535af09 --- /dev/null +++ b/workhorse/listener_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "io" + "net" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "gitlab.com/gitlab-org/gitlab/workhorse/internal/config" +) + +func TestNewListener(t *testing.T) { + const unixSocket = "testdata/sock" + + require.NoError(t, os.RemoveAll(unixSocket)) + + testCases := []struct { + network, addr string + }{ + {"tcp", "127.0.0.1:0"}, + {"unix", unixSocket}, + } + + for _, tc := range testCases { + t.Run(tc.network+"+"+tc.addr, func(t *testing.T) { + l, err := newListener("test", config.ListenerConfig{ + Addr: tc.addr, + Network: tc.network, + }) + require.NoError(t, err) + defer l.Close() + go pingServer(l) + + c, err := net.Dial(tc.network, l.Addr().String()) + require.NoError(t, err) + defer c.Close() + pingClient(t, c) + }) + } +} + +func pingServer(l net.Listener) { + c, err := l.Accept() + if err != nil { + return + } + io.WriteString(c, "ping") + c.Close() +} + +func pingClient(t *testing.T, c net.Conn) { + t.Helper() + buf, err := io.ReadAll(c) + require.NoError(t, err) + require.Equal(t, "ping", string(buf)) +} + +func TestNewListener_TLS(t *testing.T) { + const ( + certFile = "testdata/localhost.crt" + keyFile = "testdata/localhost.key" + ) + + cfg := config.ListenerConfig{Addr: "127.0.0.1:0", + Network: "tcp", + Tls: &config.TlsConfig{ + Certificate: certFile, + Key: keyFile, + }, + } + + l, err := newListener("test", cfg) + require.NoError(t, err) + defer l.Close() + go pingServer(l) + + tlsCertificate, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + + certificate, err := x509.ParseCertificate(tlsCertificate.Certificate[0]) + require.NoError(t, err) + certpool := x509.NewCertPool() + certpool.AddCert(certificate) + + c, err := tls.Dial("tcp", l.Addr().String(), &tls.Config{RootCAs: certpool}) + require.NoError(t, err) + defer c.Close() + + pingClient(t, c) +} diff --git a/workhorse/main.go b/workhorse/main.go index c0fdc10cbaf..55c29b8abf5 100644 --- a/workhorse/main.go +++ b/workhorse/main.go @@ -23,7 +23,6 @@ import ( "gitlab.com/gitlab-org/gitlab/workhorse/internal/queueing" "gitlab.com/gitlab-org/gitlab/workhorse/internal/redis" "gitlab.com/gitlab-org/gitlab/workhorse/internal/secret" - "gitlab.com/gitlab-org/gitlab/workhorse/internal/server" "gitlab.com/gitlab-org/gitlab/workhorse/internal/upstream" ) @@ -241,14 +240,20 @@ func run(boot bootConfig, cfg config.Config) error { Network: boot.listenNetwork, Addr: boot.listenAddr, } - srv := &server.Server{ - Handler: up, - Umask: boot.listenUmask, - ListenerConfigs: append(cfg.Listeners, listenerFromBootConfig), - Errors: finalErrors, + var listeners []net.Listener + oldUmask := syscall.Umask(boot.listenUmask) + for _, cfg := range append(cfg.Listeners, listenerFromBootConfig) { + l, err := newListener("upstream", cfg) + if err != nil { + return err + } + listeners = append(listeners, l) } - if err := srv.Run(); err != nil { - return fmt.Errorf("running server: %v", err) + syscall.Umask(oldUmask) + + srv := &http.Server{Handler: up} + for _, l := range listeners { + go func(l net.Listener) { finalErrors <- srv.Serve(l) }(l) } select { diff --git a/workhorse/internal/server/testdata/localhost.crt b/workhorse/testdata/localhost.crt index bee60e42e00..bee60e42e00 100644 --- a/workhorse/internal/server/testdata/localhost.crt +++ b/workhorse/testdata/localhost.crt diff --git a/workhorse/internal/server/testdata/localhost.key b/workhorse/testdata/localhost.key index b708582f02e..b708582f02e 100644 --- a/workhorse/internal/server/testdata/localhost.key +++ b/workhorse/testdata/localhost.key |