diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-30 21:10:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-10-30 21:10:11 +0300 |
commit | c51425915fb1b2c367d6d828449b5cc7772ac104 (patch) | |
tree | e0bb2d9f3e0c9aaec6fe71a9da26f9bab5d9890c | |
parent | f65227a163435d66e3f0b80f4c52ae59d8df39a2 (diff) |
Add latest changes from gitlab-org/gitlab@master
34 files changed, 579 insertions, 206 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7759cdbedc..7ef143d5525 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,15 +64,15 @@ workflow: # they serve no purpose and will run anyway when the changes are merged. - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^release-tools\/\d+\.\d+\.\d+-rc\d+$/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable(-ee)?$/ && $CI_PROJECT_PATH == "gitlab-org/gitlab"' when: never - # For merge requests running exclusively in Ruby 3.1 - - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_1/' + # For merge requests running exclusively in Ruby 3.0 + - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_0/' variables: - <<: *next-ruby-variables + <<: *default-ruby-variables PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_MERGE_REQUEST_EVENT_TYPE MR pipeline' NO_SOURCEMAPS: 'true' - if: '$CI_MERGE_REQUEST_LABELS =~ /Community contribution/' variables: - <<: *default-ruby-variables + <<: *next-ruby-variables GITLAB_DEPENDENCY_PROXY_ADDRESS: "" PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_MERGE_REQUEST_EVENT_TYPE MR pipeline (community contribution)' NO_SOURCEMAPS: 'true' @@ -83,7 +83,7 @@ workflow: # For (detached) merge request pipelines. - if: '$CI_MERGE_REQUEST_IID' variables: - <<: *default-ruby-variables + <<: *next-ruby-variables <<: *default-merge-request-slow-tests-variables PIPELINE_NAME: 'Ruby $RUBY_VERSION $CI_MERGE_REQUEST_EVENT_TYPE MR pipeline' NO_SOURCEMAPS: 'true' @@ -156,7 +156,7 @@ variables: DOCKER_VERSION: "24.0.5" RUBYGEMS_VERSION: "3.4" GO_VERSION: "1.20" - RUST_VERSION: "1.65" + RUST_VERSION: "1.73" FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json FRONTEND_FIXTURES_MAPPING_PATH: crystalball/frontend_fixtures_mapping.json diff --git a/.gitlab/ci/cng/main.gitlab-ci.yml b/.gitlab/ci/cng/main.gitlab-ci.yml index d27a4ef4b08..1ecbbbd47ad 100644 --- a/.gitlab/ci/cng/main.gitlab-ci.yml +++ b/.gitlab/ci/cng/main.gitlab-ci.yml @@ -54,6 +54,8 @@ include: GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}" GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}" RUBY_VERSION: "${FULL_RUBY_VERSION}" + NEXT_RUBY_CACHE_KEY: "${RUBY_VERSION}" + NEXT_RUBY_VERSION: "${FULL_RUBY_VERSION}" trigger: project: ${CI_PROJECT_NAMESPACE}/build/CNG-mirror branch: $TRIGGER_BRANCH diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 9dc489ab3c4..336e2df80f7 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -59,9 +59,15 @@ .if-merge-request-targeting-stable-branch: &if-merge-request-targeting-stable-branch if: '($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable(-ee|-jh)?$/' +.if-merge-request-labels-run-in-ruby3_0: &if-merge-request-labels-run-in-ruby3_0 + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_0/' + .if-merge-request-labels-run-in-ruby3_1: &if-merge-request-labels-run-in-ruby3_1 if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_1/' +.if-merge-request-labels-run-in-non-default-ruby: &if-merge-request-labels-run-in-non-default-ruby + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3_2/' + .if-merge-request-labels-as-if-foss: &if-merge-request-labels-as-if-foss if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-as-if-foss/' @@ -2738,9 +2744,9 @@ - <<: *if-default-refs changes: *code-backstage-patterns -.setup:rules:verify-ruby-3.0: +.setup:rules:verify-default-ruby: rules: - - <<: *if-merge-request-labels-run-in-ruby3_1 + - <<: *if-merge-request-labels-run-in-non-default-ruby .setup:rules:verify-tests-yml: rules: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index b652ac5e30b..cc5e9bd2985 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -51,13 +51,14 @@ gitlab_git_test: script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes -verify-ruby-3.0: +verify-default-ruby: extends: - .absolutely-predictive-job - - .setup:rules:verify-ruby-3.0 + - .setup:rules:verify-default-ruby stage: prepare script: - - echo 'Please remove label ~"pipeline:run-in-ruby3_1" so we do test against Ruby 3.0 (default version) before merging the merge request' + - echo 'Please remove label ~"pipeline:run-in-ruby3_2" so we do test against default Ruby version before merging the merge request' + - echo 'This does not work yet. See https://gitlab.com/gitlab-org/gitlab/-/issues/428537' - exit 1 verify-tests-yml: diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb index c3e8955d2a2..38841cc309d 100644 --- a/app/models/ci/catalog/resource.rb +++ b/app/models/ci/catalog/resource.rb @@ -17,8 +17,8 @@ module Ci scope :for_projects, ->(project_ids) { where(project_id: project_ids) } scope :order_by_created_at_desc, -> { reorder(created_at: :desc) } scope :order_by_created_at_asc, -> { reorder(created_at: :asc) } - scope :order_by_name_desc, -> { joins(:project).merge(Project.sorted_by_name_desc) } - scope :order_by_name_asc, -> { joins(:project).merge(Project.sorted_by_name_asc) } + scope :order_by_name_desc, -> { reorder(arel_table[:name].desc.nulls_last) } + scope :order_by_name_asc, -> { reorder(arel_table[:name].asc.nulls_last) } scope :order_by_latest_released_at_desc, -> { reorder(arel_table[:latest_released_at].desc.nulls_last) } scope :order_by_latest_released_at_asc, -> { reorder(arel_table[:latest_released_at].asc.nulls_last) } diff --git a/app/models/project.rb b/app/models/project.rb index f7e994d4beb..28b640cdf06 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -624,42 +624,6 @@ class Project < ApplicationRecord .or(arel_table[:storage_version].eq(nil))) end - scope :sorted_by_name_desc, -> { - keyset_order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :name, - column_expression: Project.arel_table[:name], - order_expression: Project.arel_table[:name].desc, - distinct: false, - nullable: :nulls_last - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Project.arel_table[:id].desc - ) - ]) - - reorder(keyset_order) - } - - scope :sorted_by_name_asc, -> { - keyset_order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :name, - column_expression: Project.arel_table[:name], - order_expression: Project.arel_table[:name].asc, - distinct: false, - nullable: :nulls_last - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Project.arel_table[:id].asc - ) - ]) - - reorder(keyset_order) - } - scope :sorted_by_updated_asc, -> { reorder(self.arel_table['updated_at'].asc) } scope :sorted_by_updated_desc, -> { reorder(self.arel_table['updated_at'].desc) } scope :sorted_by_stars_desc, -> { reorder(self.arel_table['star_count'].desc) } diff --git a/app/services/ci/pipelines/update_metadata_service.rb b/app/services/ci/pipelines/update_metadata_service.rb new file mode 100644 index 00000000000..2f2d648c13d --- /dev/null +++ b/app/services/ci/pipelines/update_metadata_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Ci + module Pipelines + class UpdateMetadataService + def initialize(pipeline, params) + @pipeline = pipeline + @params = params + end + + def execute + metadata = pipeline.pipeline_metadata + + metadata = pipeline.build_pipeline_metadata(project: pipeline.project) if metadata.nil? + + params[:name] = params[:name].strip if params.key?(:name) + + if metadata.update(params) + ServiceResponse.success(message: 'Pipeline metadata was updated', payload: pipeline) + else + ServiceResponse.error(message: 'Failed to update pipeline', payload: metadata.errors.full_messages, + reason: :bad_request) + end + end + + private + + attr_reader :pipeline, :params + end + end +end diff --git a/config/feature_flags/development/log_git_streaming_audit_events.yml b/config/feature_flags/development/log_git_streaming_audit_events.yml new file mode 100644 index 00000000000..1c3cace7aa5 --- /dev/null +++ b/config/feature_flags/development/log_git_streaming_audit_events.yml @@ -0,0 +1,8 @@ +--- +name: log_git_streaming_audit_events +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123486 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/415138 +milestone: "16.5" +type: development +group: group::source code +default_enabled: false diff --git a/db/post_migrate/20231025025733_swap_columns_for_ci_pipelines_pipeline_id_bigint_for_self_host.rb b/db/post_migrate/20231025025733_swap_columns_for_ci_pipelines_pipeline_id_bigint_for_self_host.rb new file mode 100644 index 00000000000..a960258ff3d --- /dev/null +++ b/db/post_migrate/20231025025733_swap_columns_for_ci_pipelines_pipeline_id_bigint_for_self_host.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class SwapColumnsForCiPipelinesPipelineIdBigintForSelfHost < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::Swapping + + milestone '16.6' + disable_ddl_transaction! + + TABLE_NAME = :ci_pipelines + TRIGGER_FUNCTION_NAME = :trigger_1bd97da9c1a4 + COLUMN_NAME = :auto_canceled_by_id + BIGINT_COLUMN_NAME = :auto_canceled_by_id_convert_to_bigint + FK_NAME = :fk_262d4c2d19 + BIGINT_FK_NAME = :fk_67e4288f3a + INDEX_NAME = :index_ci_pipelines_on_auto_canceled_by_id + BIGINT_INDEX_NAME = :index_ci_pipelines_on_auto_canceled_by_id_bigint + + def up + return if column_type_of?(:bigint) + + swap + end + + def down + return if column_type_of?(:integer) + + swap + end + + private + + def column_type_of?(type) + column_for(TABLE_NAME, COLUMN_NAME).sql_type.to_s == type.to_s + end + + def swap + with_lock_retries(raise_on_exhaustion: true) do + # Lock the tables involved. + lock_tables(TABLE_NAME) + + # Rename the columns to swap names + swap_columns(TABLE_NAME, COLUMN_NAME, BIGINT_COLUMN_NAME) + + # Reset the trigger function + reset_trigger_function(TRIGGER_FUNCTION_NAME) + + # Swap fkey constraint + swap_foreign_keys(TABLE_NAME, FK_NAME, BIGINT_FK_NAME) + + # Swap index + swap_indexes(TABLE_NAME, INDEX_NAME, BIGINT_INDEX_NAME) + end + end +end diff --git a/db/schema_migrations/20231025025733 b/db/schema_migrations/20231025025733 new file mode 100644 index 00000000000..a488c5206e1 --- /dev/null +++ b/db/schema_migrations/20231025025733 @@ -0,0 +1 @@ +c0129899dcea5f304661b49665a371de86dbff9df88afbb3fdbb348a411c1dd8
\ No newline at end of file diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index e634534509b..50616974ae1 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -518,3 +518,57 @@ DELETE /projects/:id/pipelines/:pipeline_id ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" --request "DELETE" "https://gitlab.example.com/api/v4/projects/1/pipelines/46" ``` + +## Update pipeline metadata + +You can update the metadata of a pipeline. The metadata contains the name of the pipeline. + +```plaintext +PUT /projects/:id/pipelines/:pipeline_id/metadata +``` + +| Attribute | Type | Required | Description | +|---------------|----------------|----------|-------------| +| `id` | integer/string | Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | +| `pipeline_id` | integer | Yes | The ID of a pipeline | +| `name` | string | Yes | The new name of the pipeline | + +Sample request: + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" --data "name=Some new pipeline name" "https://gitlab.example.com/api/v4/projects/1/pipelines/46/metadata" +``` + +Sample response: + +```json +{ + "id": 46, + "iid": 11, + "project_id": 1, + "status": "running", + "ref": "main", + "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", + "before_sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", + "tag": false, + "yaml_errors": null, + "user": { + "name": "Administrator", + "username": "root", + "id": 1, + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://localhost:3000/root" + }, + "created_at": "2016-08-11T11:28:34.085Z", + "updated_at": "2016-08-11T11:32:35.169Z", + "started_at": null, + "finished_at": "2016-08-11T11:32:35.145Z", + "committed_at": null, + "duration": null, + "queued_duration": 0.010, + "coverage": null, + "web_url": "https://example.com/foo/bar/pipelines/46", + "name": "Some new pipeline name" +} +``` diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index a335794b209..389a79c2596 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -22,6 +22,7 @@ You can use a GitLab CI/CD job token to authenticate with specific API endpoints - [Get job token's job](../../api/jobs.md#get-job-tokens-job). - [Pipeline triggers](../../api/pipeline_triggers.md), using the `token=` parameter to [trigger a multi-project pipeline](../pipelines/downstream_pipelines.md#trigger-a-multi-project-pipeline-by-using-the-api). +- [Update pipeline metadata](../../api/pipelines.md#update-pipeline-metadata) - [Releases](../../api/releases/index.md) and [Release links](../../api/releases/links.md). - [Terraform plan](../../user/infrastructure/index.md). - [Deployments](../../api/deployments.md). diff --git a/doc/development/pipelines/index.md b/doc/development/pipelines/index.md index d22c01972f5..77f91300a57 100644 --- a/doc/development/pipelines/index.md +++ b/doc/development/pipelines/index.md @@ -610,15 +610,26 @@ Exceptions to this general guideline should be motivated and documented. ### Ruby versions testing -We're running Ruby 3.0 on GitLab.com, as well as for merge requests and the default branch. -To prepare for the next release, Ruby 3.1, we also run our test suite against Ruby 3.1 on -a dedicated 2-hourly scheduled pipelines. +We're running Ruby 3.0 on GitLab.com, as well as for the default branch. +To prepare for the next Ruby version, we run merge requests in Ruby 3.1. -For merge requests, you can add the `pipeline:run-in-ruby3_1` label to switch -the Ruby version used for running the whole test suite to 3.1. When you do -this, the test suite will no longer run in Ruby 3.0 (default), and an -additional job `verify-ruby-3.0` will also run and always fail to remind us to -remove the label and run in Ruby 3.0 before merging the merge request. +This takes effects at the time when +[Run merge requests in Ruby 3.1 by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134290) +is merged. See +[Ruby 3.1 epic](https://gitlab.com/groups/gitlab-org/-/epics/10034) +for the roadmap to fully make Ruby 3.1 the default. + +To make sure both Ruby versions are working, we also run our test suite +against both Ruby 3.0 and Ruby 3.1 on dedicated 2-hourly scheduled pipelines. + +For merge requests, you can add the `pipeline:run-in-ruby3_0` label to switch +the Ruby version to 3.0. When you do this, the test suite will no longer run +in Ruby 3.1 (default for merge requests). + +When the pipeline is running in a Ruby version not considered default, an +additional job `verify-default-ruby` will also run and always fail to remind +us to remove the label and run in default Ruby before merging the merge +request. At the moment both Ruby 3.0 and Ruby 3.1 are considered default. This should let us: @@ -638,7 +649,7 @@ We also run our test suite against PostgreSQL 13 upon specific database library | Where? | PostgreSQL version | Ruby version | |--------------------------------------------------------------------------------------------------|-------------------------------------------------|-----------------------| -| Merge requests | 14 (default version), 13 for DB library changes | 3.0 (default version) | +| Merge requests | 14 (default version), 13 for DB library changes | 3.1 | | `master` branch commits | 14 (default version), 13 for DB library changes | 3.0 (default version) | | `maintenance` scheduled pipelines for the `master` branch (every even-numbered hour) | 14 (default version), 13 for DB library changes | 3.0 (default version) | | `maintenance` scheduled pipelines for the `ruby3_1` branch (every odd-numbered hour), see below. | 14 (default version), 13 for DB library changes | 3.1 | diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md index df1e1f49083..1531e01577f 100644 --- a/doc/integration/advanced_search/elasticsearch_troubleshooting.md +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -519,3 +519,13 @@ unexpectedly high `buff/cache` usage. When you reindex, you might get a `Couldn't load task status` error. A `sliceId must be greater than 0 but was [-1]` error might also appear on the Elasticsearch host. As a workaround, consider [reindexing from scratch](../../integration/advanced_search/elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index) or upgrading to GitLab 16.3. For more information, see [issue 422938](https://gitlab.com/gitlab-org/gitlab/-/issues/422938). + +## Migration `BackfillProjectPermissionsInBlobs` has been halted in GitLab 15.11 + +In GitLab 15.11, it is possible for the `BackfillProjectPermissionsInBlobs` migration to be halted with the following error message in the `elasticsearch.log`: + +```shell +migration has failed with NoMethodError:undefined method `<<' for nil:NilClass, no retries left +``` + +If `BackfillProjectPermissionsInBlobs` is the only halted migration, you can upgrade to the latest patch version of GitLab 16.0, which includes [the fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118494). Otherwise, you can ignore the error as it will not affect the current functionality of advanced search. diff --git a/lib/api/api.rb b/lib/api/api.rb index 8a26ae7e6f6..7a71ec416fd 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -387,6 +387,7 @@ module API mount ::API::Internal::MailRoom mount ::API::Internal::ContainerRegistry::Migration mount ::API::Internal::Workhorse + mount ::API::Internal::Shellhorse version 'v3', using: :path do # Although the following endpoints are kept behind V3 namespace, diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index bd5c04f401b..3361f4564b2 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -288,6 +288,33 @@ module API end end + desc 'Updates pipeline metadata' do + detail 'This feature was introduced in GitLab 16.6' + success status: 200, model: Entities::Ci::PipelineWithMetadata + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID', documentation: { example: 18 } + requires :name, type: String, desc: 'The name of the pipeline', documentation: { example: 'Deployment to production' } + end + route_setting :authentication, job_token_allowed: true + put ':id/pipelines/:pipeline_id/metadata', urgency: :low, feature_category: :continuous_integration do + authorize! :update_pipeline, pipeline + + response = ::Ci::Pipelines::UpdateMetadataService.new(pipeline, params.slice(:name)).execute + + if response.success? + present response.payload, with: Entities::Ci::PipelineWithMetadata + else + render_api_error_with_reason!(response.reason, response.message, response.payload.join(', ')) + end + end + desc 'Retry builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' success status: 201, model: Entities::Ci::Pipeline diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 56b157f662a..dbeacda01ea 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -337,6 +337,12 @@ module API unauthorized! end + def authenticate_by_gitlab_shell_or_workhorse_token! + return require_gitlab_workhorse! unless headers[GITLAB_SHELL_API_HEADER].present? + + authenticate_by_gitlab_shell_token! + end + def authenticated_with_can_read_all_resources! authenticate! forbidden! unless current_user.can_read_all_resources? diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index f66f899c98b..0c5b12d48e9 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -123,6 +123,10 @@ module API # Defined in EE end + def need_git_audit_event? + false + end + private def repository_path diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index f9dc888fbeb..cebb1390acc 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -66,7 +66,8 @@ module API git_config_options: ["uploadpack.allowFilter=true", "uploadpack.allowAnySHA1InWant=true"], gitaly: gitaly_payload(params[:action]), - gl_console_messages: check_result.console_messages + gl_console_messages: check_result.console_messages, + need_audit: need_git_audit_event? }.merge!(actor.key_details) # Custom option for git-receive-pack command @@ -77,7 +78,9 @@ module API payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end - send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + unless Feature.enabled?(:log_git_streaming_audit_events, project) + send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + end response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction diff --git a/lib/api/internal/shellhorse.rb b/lib/api/internal/shellhorse.rb new file mode 100644 index 00000000000..89210c8a78a --- /dev/null +++ b/lib/api/internal/shellhorse.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module API + module Internal + class Shellhorse < ::API::Base + before { authenticate_by_gitlab_shell_or_workhorse_token! } + + helpers ::API::Helpers::InternalHelpers + + COMMANDS_TO_AUDIT = %w[git-upload-pack git-receive-pack].freeze + + helpers do + def check_clone_or_pull_or_push_verb(params) + return 'push' if params[:action] == 'git-receive-pack' + + # we must set the default value for wants/haves because + # gitlab shell/workhorse will trim the whole posted params + # json key if its value is 0 + wants = haves = 0 + if params.key?(:packfile_stats) + wants = Integer(params[:packfile_stats][:wants]) if params[:packfile_stats][:wants].present? + haves = Integer(params[:packfile_stats][:haves]) if params[:packfile_stats][:haves].present? + end + + wants > 0 && haves == 0 ? 'clone' : 'pull' + end + end + + namespace 'internal' do + namespace 'shellhorse' do + params do + requires :action, type: String + requires :protocol, type: String + requires :gl_repository, type: String # repository identifier, such as project-7 + optional :packfile_stats, type: Hash do + # wants is the number of objects the client announced it wants. + optional :wants, type: Integer + # haves is the number of objects the client announced it has. + optional :haves, type: Integer + end + end + + post '/git_audit_event', feature_category: :source_code_management do + unless COMMANDS_TO_AUDIT.include?(params[:action]) + break response_with_status(code: 400, success: false, message: "No valid action specified") + end + + check_result = access_check_result + break check_result if unsuccessful_response?(check_result) + + unless need_git_audit_event? + break response_with_status(code: 200, success: false, message: "No git audit event needed") + end + + unless check_result.is_a?(::Gitlab::GitAccessResult::Success) + break response_with_status(code: 500, success: false, + message: ::API::Helpers::InternalHelpers::UNKNOWN_CHECK_RESULT_ERROR) + end + + msg = { + protocol: params[:protocol], + action: params[:action], + verb: check_clone_or_pull_or_push_verb(params) + } + send_git_audit_streaming_event(msg) + response_with_status(message: msg) + end + end + end + end + end +end + +API::Internal::Shellhorse.prepend_mod_with('API::Internal::Shellhorse') diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb index 732e0e14257..b007a957348 100644 --- a/lib/gitlab/git_access_project.rb +++ b/lib/gitlab/git_access_project.rb @@ -47,7 +47,7 @@ module Gitlab end def repository_path_match - strong_memoize(:repository_path_match) { repository_path.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } + strong_memoize(:repository_path_match) { repository_path&.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } end def ensure_project_on_push! diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb deleted file mode 100644 index b8365bdf41f..00000000000 --- a/lib/gitlab/git_audit_event.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass - attr_reader :project, :user, :author - - def initialize(player, project) - @project = project - @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player - @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player - end - - def send_audit_event(msg) - return if user.blank? || project.blank? - - audit_context = { - name: 'repository_git_operation', - stream_only: true, - author: author, - scope: project, - target: project, - message: msg - } - - ::Gitlab::Audit::Auditor.audit(audit_context) - end - end -end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f2db7e3c9b9..057e89a2a97 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -20,7 +20,7 @@ module Gitlab include JwtAuthenticatable class << self - def git_http_ok(repository, repo_type, user, action, show_all_refs: false) + def git_http_ok(repository, repo_type, user, action, show_all_refs: false, need_audit: false) raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) attrs = { @@ -28,6 +28,7 @@ module Gitlab GL_REPOSITORY: repo_type.identifier_for_container(repository.container), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, + NeedAudit: need_audit, Repository: repository.gitaly_repository.to_h, GitConfigOptions: [], GitalyServer: { diff --git a/package.json b/package.json index 927102f2afb..2d6c590930d 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@gitlab/cluster-client": "^2.1.0", "@gitlab/favicon-overlay": "2.0.0", "@gitlab/fonts": "^1.3.0", - "@gitlab/svgs": "3.68.0", + "@gitlab/svgs": "3.69.0", "@gitlab/ui": "66.37.0", "@gitlab/visual-review-tools": "1.7.3", "@gitlab/web-ide": "0.0.1-dev-20231004090414", diff --git a/spec/lib/api/helpers_spec.rb b/spec/lib/api/helpers_spec.rb index 5d343ec2777..21b3b8e6927 100644 --- a/spec/lib/api/helpers_spec.rb +++ b/spec/lib/api/helpers_spec.rb @@ -1327,4 +1327,79 @@ RSpec.describe API::Helpers, feature_category: :shared do end end end + + describe '#authenticate_by_gitlab_shell_or_workhorse_token!' do + include GitlabShellHelpers + include WorkhorseHelpers + + include_context 'workhorse headers' + + let(:headers) { {} } + let(:params) { {} } + + context 'when request from gitlab shell' do + let(:valid_secret_token) { 'valid' } + let(:invalid_secret_token) { 'invalid' } + + before do + allow(helper).to receive_messages(headers: headers) + end + + context 'with invalid token' do + let(:headers) { gitlab_shell_internal_api_request_header(secret_token: invalid_secret_token) } + + it 'unauthorized' do + expect(helper).to receive(:unauthorized!) + + helper.authenticate_by_gitlab_shell_or_workhorse_token! + end + end + + context 'with valid token' do + let(:headers) { gitlab_shell_internal_api_request_header } + + it 'authorized' do + expect(helper).not_to receive(:unauthorized!) + + helper.authenticate_by_gitlab_shell_or_workhorse_token! + end + end + end + + context 'when request from gitlab workhorse' do + let(:env) { {} } + let(:request) { ActionDispatch::Request.new(env) } + + before do + allow_any_instance_of(ActionDispatch::Request).to receive(:headers).and_return(headers) + allow(helper).to receive(:request).and_return(request) + allow(helper).to receive_messages(params: params, headers: headers, env: env) + end + + context 'with invalid token' do + let(:headers) { { Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => JWT.encode({ 'iss' => 'gitlab-workhorse' }, 'wrongkey', 'HS256') } } + + before do + allow(JWT).to receive(:decode).and_return([{ 'iss' => 'gitlab-workhorse' }]) + end + + it 'unauthorized' do + expect(helper).to receive(:forbidden!) + + helper.authenticate_by_gitlab_shell_or_workhorse_token! + end + end + + context 'with valid token' do + let(:headers) { workhorse_headers } + let(:env) { { 'HTTP_GITLAB_WORKHORSE' => 1 } } + + it 'authorized' do + expect(helper).not_to receive(:forbidden!) + + helper.authenticate_by_gitlab_shell_or_workhorse_token! + end + end + end + end end diff --git a/spec/lib/gitlab/git_audit_event_spec.rb b/spec/lib/gitlab/git_audit_event_spec.rb deleted file mode 100644 index c533b39f550..00000000000 --- a/spec/lib/gitlab/git_audit_event_spec.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GitAuditEvent, feature_category: :source_code_management do - let_it_be(:player) { create(:user) } - let_it_be(:group) { create(:group, :public) } - let_it_be(:project) { create(:project) } - - subject { described_class.new(player, project) } - - describe '#send_audit_event' do - let(:msg) { 'valid_msg' } - - context 'with successfully sending' do - let_it_be(:project) { create(:project, namespace: group) } - - before do - allow(::Gitlab::Audit::Auditor).to receive(:audit) - end - - context 'when player is a regular user' do - it 'sends git audit event' do - expect(::Gitlab::Audit::Auditor).to receive(:audit).with(a_hash_including( - name: 'repository_git_operation', - stream_only: true, - author: player, - scope: project, - target: project, - message: msg - )).once - - subject.send_audit_event(msg) - end - end - - context 'when player is ::API::Support::GitAccessActor' do - let_it_be(:user) { player } - let_it_be(:key) { create(:key, user: user) } - let_it_be(:git_access_actor) { ::API::Support::GitAccessActor.new(user: user, key: key) } - - subject { described_class.new(git_access_actor, project) } - - it 'sends git audit event' do - expect(::Gitlab::Audit::Auditor).to receive(:audit).with(a_hash_including( - name: 'repository_git_operation', - stream_only: true, - author: git_access_actor.deploy_key_or_user, - scope: project, - target: project, - message: msg - )).once - - subject.send_audit_event(msg) - end - end - end - - context 'when user is blank' do - let_it_be(:player) { nil } - - it 'does not send git audit event' do - expect(::Gitlab::Audit::Auditor).not_to receive(:audit) - - subject.send_audit_event(msg) - end - end - - context 'when project is blank' do - let_it_be(:project) { nil } - - it 'does not send git audit event' do - expect(::Gitlab::Audit::Auditor).not_to receive(:audit) - - subject.send_audit_event(msg) - end - end - end -end diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index cca18cb05c7..d77763f89be 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -226,7 +226,8 @@ RSpec.describe Gitlab::Workhorse, feature_category: :shared do GL_ID: "user-#{user.id}", GL_USERNAME: user.username, GL_REPOSITORY: "project-#{project.id}", - ShowAllRefs: false + ShowAllRefs: false, + NeedAudit: false } end @@ -277,6 +278,12 @@ RSpec.describe Gitlab::Workhorse, feature_category: :shared do it { is_expected.to include(ShowAllRefs: true) } end + context 'need_audit enabled' do + subject { described_class.git_http_ok(repository, Gitlab::GlRepository::PROJECT, user, action, show_all_refs: true, need_audit: true) } + + it { is_expected.to include(NeedAudit: true) } + end + context 'when a feature flag is set for a single project' do before do stub_feature_flags(gitaly_mep_mep: project) diff --git a/spec/models/ci/catalog/resource_spec.rb b/spec/models/ci/catalog/resource_spec.rb index 4e292fc0ec0..cb9152cfd74 100644 --- a/spec/models/ci/catalog/resource_spec.rb +++ b/spec/models/ci/catalog/resource_spec.rb @@ -53,19 +53,31 @@ RSpec.describe Ci::Catalog::Resource, feature_category: :pipeline_composition do end describe '.order_by_name_desc' do - it 'returns catalog resources sorted by descending name' do - ordered_resources = described_class.order_by_name_desc + subject(:ordered_resources) { described_class.order_by_name_desc } + it 'returns catalog resources sorted by descending name' do expect(ordered_resources.pluck(:name)).to eq(%w[Z L A]) end + + it 'returns catalog resources sorted by descending name with nulls last' do + resource.update!(name: nil) + + expect(ordered_resources.pluck(:name)).to eq(['Z', 'L', nil]) + end end describe '.order_by_name_asc' do - it 'returns catalog resources sorted by ascending name' do - ordered_resources = described_class.order_by_name_asc + subject(:ordered_resources) { described_class.order_by_name_asc } + it 'returns catalog resources sorted by ascending name' do expect(ordered_resources.pluck(:name)).to eq(%w[A L Z]) end + + it 'returns catalog resources sorted by ascending name with nulls last' do + resource.update!(name: nil) + + expect(ordered_resources.pluck(:name)).to eq(['L', 'Z', nil]) + end end describe '.order_by_latest_released_at_desc' do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 5acf03e09d7..d4e23823f23 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2127,28 +2127,6 @@ RSpec.describe Project, factory_default: :keep, feature_category: :groups_and_pr end end - describe 'sorting by name' do - let_it_be(:project1) { create(:project, name: 'A') } - let_it_be(:project2) { create(:project, name: 'Z') } - let_it_be(:project3) { create(:project, name: 'L') } - - context 'when using .sort_by_name_desc' do - it 'reorders the projects by descending name order' do - projects = described_class.sorted_by_name_desc - - expect(projects.pluck(:name)).to eq(%w[Z L A]) - end - end - - context 'when using .sort_by_name_asc' do - it 'reorders the projects by ascending name order' do - projects = described_class.sorted_by_name_asc - - expect(projects.pluck(:name)).to eq(%w[A L Z]) - end - end - end - describe '.with_shared_runners_enabled' do subject { described_class.with_shared_runners_enabled } diff --git a/spec/requests/api/ci/pipelines_spec.rb b/spec/requests/api/ci/pipelines_spec.rb index 3544a6dd72a..f823da9fb2d 100644 --- a/spec/requests/api/ci/pipelines_spec.rb +++ b/spec/requests/api/ci/pipelines_spec.rb @@ -1107,6 +1107,78 @@ RSpec.describe API::Ci::Pipelines, feature_category: :continuous_integration do end end + describe 'PUT /projects/:id/pipelines/:pipeline_id/name' do + let_it_be(:pipeline_creator) { create(:user) } + let(:pipeline) { create(:ci_pipeline, project: project, user: pipeline_creator) } + let(:name) { 'A new pipeline name' } + + subject(:execute) do + put api("/projects/#{project.id}/pipelines/#{pipeline.id}/metadata", current_user), params: { name: name } + end + + context 'authorized user' do + let(:current_user) { create(:user) } + + before do + project.add_developer(current_user) + end + + it 'renames pipeline when name is valid', :aggregate_failures do + expect { execute }.to change { pipeline.reload.name }.to(name) + expect(response).to have_gitlab_http_status(:ok) + end + + context 'when name is invalid' do + let(:name) { 'a' * 256 } + + it 'does not rename pipeline', :aggregate_failures do + expect { execute }.not_to change { pipeline.reload.name } + expect(response).to have_gitlab_http_status(:bad_request) + expect(json_response['message']).to eq('Failed to update pipeline - Name is too long (maximum is 255 characters)') + end + end + end + + context 'unauthorized user' do + let(:current_user) { create(:user) } + + context 'when user is not a member' do + it 'does not rename pipeline', :aggregate_failures do + expect { execute }.not_to change { pipeline.reload.name } + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'when user is a member' do + before do + project.add_reporter(current_user) + end + + it 'does not rename pipeline', :aggregate_failures do + expect { execute }.not_to change { pipeline.reload.name } + expect(response).to have_gitlab_http_status(:forbidden) + end + end + end + + context 'when authorized with job token' do + let(:job) { create(:ci_build, :running, pipeline: pipeline, project: project, user: pipeline.user) } + + before do + project.add_developer(pipeline.user) + end + + subject(:execute) do + put api("/projects/#{project.id}/pipelines/#{pipeline.id}/metadata", nil, job_token: job.token), params: { name: name } + end + + it 'renames pipeline when name is valid', :aggregate_failures do + expect { execute }.to change { pipeline.reload.name }.to(name) + expect(response).to have_gitlab_http_status(:ok) + end + end + end + describe 'POST /projects/:id/pipelines/:pipeline_id/retry' do context 'authorized user' do let_it_be(:pipeline) do diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index cf0cd9a2e85..95d620a55ae 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -744,6 +744,17 @@ RSpec.describe API::Internal::Base, feature_category: :system_access do expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-mep-mep' => 'false') end end + + context 'with audit event' do + it 'does not send a git streaming audit event' do + expect(::Gitlab::Audit::Auditor).not_to receive(:audit) + + pull(key, project) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response["need_audit"]).to be_falsy + end + end end context "git push" do @@ -757,6 +768,7 @@ RSpec.describe API::Internal::Base, feature_category: :system_access do expect(json_response["gl_project_path"]).to eq(project.full_path) expect(json_response["gl_key_type"]).to eq("key") expect(json_response["gl_key_id"]).to eq(key.id) + expect(json_response["need_audit"]).to be_falsy expect(json_response["gitaly"]).not_to be_nil expect(json_response["gitaly"]["repository"]).not_to be_nil expect(json_response["gitaly"]["repository"]["storage_name"]).to eq(project.repository.gitaly_repository.storage_name) diff --git a/spec/services/ci/pipelines/update_metadata_service_spec.rb b/spec/services/ci/pipelines/update_metadata_service_spec.rb new file mode 100644 index 00000000000..939ce7f5785 --- /dev/null +++ b/spec/services/ci/pipelines/update_metadata_service_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Pipelines::UpdateMetadataService, feature_category: :continuous_integration do + subject(:execute) { described_class.new(pipeline, { name: name }).execute } + + let(:name) { 'Some random pipeline name' } + + context 'when pipeline has no name' do + let(:pipeline) { create(:ci_pipeline) } + + it 'updates the name' do + expect { execute }.to change { pipeline.reload.name }.to(name) + end + end + + context 'when pipeline has a name' do + let(:pipeline) { create(:ci_pipeline, name: 'Some other name') } + + it 'updates the name' do + expect { execute }.to change { pipeline.reload.name }.to(name) + end + end + + context 'when new name is too long' do + let(:pipeline) { create(:ci_pipeline) } + let(:name) { 'a' * 256 } + + it 'does not update the name' do + expect { execute }.not_to change { pipeline.reload.name } + end + end +end diff --git a/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb b/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb index 2c2be0152a0..f91cf22f27e 100644 --- a/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb +++ b/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb @@ -14,7 +14,7 @@ RSpec.shared_examples 'sends git audit streaming event' do let(:project) { create(:project, :public, :repository, namespace: group) } before do - group.external_audit_event_destinations.create!(destination_url: 'http://example.com') + create(:external_audit_event_destination, group: group) project.add_developer(user) end @@ -38,7 +38,7 @@ RSpec.shared_examples 'sends git audit streaming event' do let(:project) { create(:project, :private, :repository, namespace: group) } before do - group.external_audit_event_destinations.create!(destination_url: 'http://example.com') + create(:external_audit_event_destination, group: group) project.add_developer(user) sign_in(user) end @@ -52,9 +52,40 @@ RSpec.shared_examples 'sends git audit streaming event' do request.headers.merge! auth_env(user.username, password, nil) end end - it 'sends the audit streaming event' do - expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async).once - subject + + context 'when log_git_streaming_audit_events is enable' do + it 'does not send the audit streaming event' do + expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async) + subject + end + + it 'respond the need audit to be true' do + subject + + expect(response).to have_gitlab_http_status(:ok) + + audit_flag = json_response["need_audit"] || json_response["NeedAudit"] + expect(audit_flag).to be_truthy + end + end + + context 'when log_git_streaming_audit_events is disable' do + before do + stub_feature_flags(log_git_streaming_audit_events: false) + end + + it "sends git streaming audit event" do + expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async).once + + subject + end + + it 'respond the need audit to be false' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response["need_audit"]).to be_falsy + end end end end diff --git a/yarn.lock b/yarn.lock index 6042e20c0bb..2177034ebab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1269,10 +1269,10 @@ stylelint-declaration-strict-value "1.9.2" stylelint-scss "5.1.0" -"@gitlab/svgs@3.68.0": - version "3.68.0" - resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.68.0.tgz#11de74e9ea9c045710bc9c4a0a29b57b8127ee41" - integrity sha512-DMEJ/wAb/kYvgitZwKl2Z0ro7Z2vqVSf5dStEnNFKYGp4R8mYvXAixE6aPWsCPWFRXeylmfdd828xUryrCMXrw== +"@gitlab/svgs@3.69.0": + version "3.69.0" + resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.69.0.tgz#bf76b8ffbe72a783807761a38abe8aaedcfe8c12" + integrity sha512-Zu8Fcjhi3Bk26jZOptcD5F4SHWC7/KuAe00NULViCeswKdoda1k19B+9oCSbsbxY7vMoFuD20kiCJdBCpxb3HA== "@gitlab/ui@66.37.0": version "66.37.0" |