diff options
Diffstat (limited to 'lib/api')
49 files changed, 580 insertions, 416 deletions
diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 94c1942a244..542b2390df2 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -26,7 +26,7 @@ module API def get_runner_details_from_request return get_runner_ip unless params['info'].present? - attributes_for_keys(%w(name version revision platform architecture executor), params['info']) + attributes_for_keys(%w[name version revision platform architecture executor], params['info']) .merge(get_system_id_from_request) .merge(get_runner_config_from_request) .merge(get_runner_ip) @@ -45,9 +45,7 @@ module API def current_runner token = params[:token] - if token - ::Ci::Runner.sticking.stick_or_unstick_request(env, :runner, token) - end + load_balancer_stick_request(::Ci::Runner, :runner, token) if token strong_memoize(:current_runner) do ::Ci::Runner.find_by_token(token.to_s) @@ -111,11 +109,7 @@ module API def current_job id = params[:id] - if id - ::Ci::Build - .sticking - .stick_or_unstick_request(env, :build, id) - end + load_balancer_stick_request(::Ci::Build, :build, id) if id strong_memoize(:current_job) do ::Ci::Build.find_by_id(id) @@ -155,7 +149,7 @@ module API private def get_runner_config_from_request - { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } + { config: attributes_for_keys(%w[gpus], params.dig('info', 'config')) } end def metrics diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 5d60c004a03..6f0a2ff7f62 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -250,7 +250,7 @@ module API ] end route_setting :authentication, job_token_allowed: true - get '/allowed_agents', urgency: :low, feature_category: :deployment_management do + get '/allowed_agents', urgency: :default, feature_category: :deployment_management do validate_current_authenticated_job status 200 diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 531235dc9b2..acb64cd0d3a 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -61,7 +61,7 @@ module API requires :sha, type: String, desc: 'The commit hash', documentation: { example: '18f3e63d05582537db6d183d9d557be09e1f90c8' } requires :state, type: String, desc: 'The state of the status', - values: %w(pending running success failed canceled), + values: %w[pending running success failed canceled], documentation: { example: 'pending' } optional :ref, type: String, desc: 'The ref', documentation: { example: 'develop' } @@ -80,75 +80,16 @@ module API post ':id/statuses/:sha' do authorize! :create_commit_status, user_project - not_found! 'Commit' unless commit + response = + ::Ci::CreateCommitStatusService + .new(user_project, current_user, params) + .execute(optional_commit_status_params: optional_commit_status_params) - # Since the CommitStatus is attached to ::Ci::Pipeline (in the future Pipeline) - # We need to always have the pipeline object - # To have a valid pipeline object that can be attached to specific MR - # Other CI service needs to send `ref` - # If we don't receive it, we will attach the CommitStatus to - # the first found branch on that commit - - pipeline = all_matching_pipelines.first - - ref = params[:ref] - ref ||= pipeline&.ref - ref ||= user_project.repository.branch_names_contains(commit.sha).first - not_found! 'References for commit' unless ref - - name = params[:name] || params[:context] || 'default' - - pipeline ||= user_project.ci_pipelines.build( - source: :external, - sha: commit.sha, - ref: ref, - user: current_user, - protected: user_project.protected_for?(ref)) - - pipeline.ensure_project_iid! - pipeline.save! - - authorize! :update_pipeline, pipeline - - # rubocop: disable Performance/ActiveRecordSubtransactionMethods - stage = pipeline.stages.safe_find_or_create_by!(name: 'external') do |stage| - stage.position = GenericCommitStatus::EXTERNAL_STAGE_IDX - stage.project = pipeline.project - end - # rubocop: enable Performance/ActiveRecordSubtransactionMethods - - status = GenericCommitStatus.running_or_pending.find_or_initialize_by( - project: user_project, - pipeline: pipeline, - name: name, - ref: ref, - user: current_user, - protected: user_project.protected_for?(ref), - ci_stage: stage, - stage_idx: stage.position, - stage: 'external' - ) - - updatable_optional_attributes = %w[target_url description coverage] - status.assign_attributes(attributes_for_keys(updatable_optional_attributes)) - - render_validation_error!(status) unless status.valid? - - response = ::Ci::Pipelines::AddJobService.new(pipeline).execute!(status) do |job| - apply_job_state!(job) - rescue ::StateMachines::InvalidTransition => e - render_api_error!(e.message, 400) + if response.error? + render_api_error!(response.message, response.http_status) + else + present response.payload[:job], with: Entities::CommitStatus end - - render_validation_error!(response.payload[:job]) unless response.success? - - if pipeline.latest? - MergeRequest - .where(source_project: user_project, source_branch: ref) - .update_all(head_pipeline_id: pipeline.id) - end - - present response.payload[:job], with: Entities::CommitStatus end # rubocop: enable CodeReuse/ActiveRecord @@ -183,6 +124,11 @@ module API render_api_error!('invalid state', 400) end end + + def optional_commit_status_params + updatable_optional_attributes = %w[target_url description coverage] + attributes_for_keys(updatable_optional_attributes) + end end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index a4e1e8308c3..069d117db17 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -41,7 +41,7 @@ module API namespace: namespace, user: current_user, label: 'counts.web_ide_commits', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis, key_path: 'counts.web_ide_commits').to_context] + context: [Gitlab::Usage::MetricDefinition.context_for('counts.web_ide_commits').to_context] ) end end diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index a045a3d4828..4278510e999 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -197,7 +197,7 @@ module API route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] - packages = + available_packages = if Feature.enabled?(:npm_allow_packages_in_multiple_projects) finder_for_endpoint_scope(package_name).execute else @@ -205,7 +205,8 @@ module API .execute end - redirect_request = project_or_nil.blank? || packages.empty? + # In order to redirect a request, packages should not exist (without taking the user into account). + redirect_request = project_or_nil.blank? || available_packages.empty? redirect_registry_request( forward_to_registry: redirect_request, @@ -213,9 +214,25 @@ module API target: project_or_nil, package_name: package_name ) do - authorize_read_package!(project) + if endpoint_scope == :project || Feature.disabled?(:npm_allow_packages_in_multiple_projects) + authorize_read_package!(project) + elsif Feature.enabled?(:npm_allow_packages_in_multiple_projects) + available_packages_to_user = ::Packages::Npm::PackagesForUserFinder.new( + current_user, + group_or_namespace, + package_name: params[:package_name] + ).execute + + if available_packages.any? && available_packages_to_user.empty? + forbidden! if current_user + + not_found!('Packages') + end + + available_packages = available_packages_to_user + end - not_found!('Packages') if packages.empty? + not_found!('Packages') if available_packages.empty? if endpoint_scope == :project && Feature.enabled?(:npm_metadata_cache, project) if metadata_cache&.file&.exists? @@ -228,7 +245,7 @@ module API enqueue_sync_metadata_cache_worker(project, package_name) end - metadata = generate_metadata_service(packages).execute.payload + metadata = generate_metadata_service(available_packages).execute.payload present metadata, with: ::API::Entities::NpmPackage end end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 45466a1894c..7c608f6f78e 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -76,7 +76,7 @@ module API requires :base_sha, type: String, desc: 'Base commit SHA in the source branch' requires :start_sha, type: String, desc: 'SHA referencing commit in target branch' requires :head_sha, type: String, desc: 'SHA referencing HEAD of this merge request' - requires :position_type, type: String, desc: 'Type of the position reference', values: %w(text image) + requires :position_type, type: String, desc: 'Type of the position reference', values: %w[text image file] optional :new_path, type: String, desc: 'File path after change' optional :new_line, type: Integer, desc: 'Line number after change' optional :old_path, type: String, desc: 'File path before change' diff --git a/lib/api/entities/ci/job_request/job_info.rb b/lib/api/entities/ci/job_request/job_info.rb index 5c3f4b08af2..e228e490946 100644 --- a/lib/api/entities/ci/job_request/job_info.rb +++ b/lib/api/entities/ci/job_request/job_info.rb @@ -7,6 +7,8 @@ module API class JobInfo < Grape::Entity expose :id, :name, :stage expose :project_id, :project_name + expose :time_in_queue_seconds + expose :project_jobs_running_on_instance_runners_count end end end diff --git a/lib/api/entities/diff.rb b/lib/api/entities/diff.rb index e9650f07f00..b9538893d32 100644 --- a/lib/api/entities/diff.rb +++ b/lib/api/entities/diff.rb @@ -5,7 +5,7 @@ module API class Diff < Grape::Entity expose :json_safe_diff, as: :diff, documentation: { type: 'string', - example: '--- a/doc/update/5.4-to-6.0.md\n+++ b/doc/update/5.4-to-6.0.md\n@@ -71,6 +71,8 @@\n...' + example: '@@ -71,6 +71,8 @@\n...' } expose :new_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } expose :old_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } diff --git a/lib/api/entities/feature.rb b/lib/api/entities/feature.rb index 48dd5a22a7e..f341472e8c2 100644 --- a/lib/api/entities/feature.rb +++ b/lib/api/entities/feature.rb @@ -7,8 +7,10 @@ module API expose :state, documentation: { type: 'string', example: 'off' } expose :gates, using: Entities::FeatureGate do |model| model.gates.map do |gate| - value = model.gate_values[gate.key] - + # in Flipper 0.26.1, they removed two GateValues#[] method calls for performance reasons + # https://github.com/flippercloud/flipper/pull/706/commits/ed914b6adc329455a634be843c38db479299efc7 + # https://github.com/flippercloud/flipper/commit/eee20f3ae278d168c8bf70a7a5fcc03bedf432b5 + value = model.gate_values.send(gate.key) # rubocop:disable GitlabSecurity/PublicSend # By default all gate values are populated. Only show relevant ones. if (value.is_a?(Integer) && value == 0) || (value.is_a?(Set) && value.empty?) next diff --git a/lib/api/entities/feature_flag/basic_user_list.rb b/lib/api/entities/feature_flag/basic_user_list.rb new file mode 100644 index 00000000000..df577e9f1a4 --- /dev/null +++ b/lib/api/entities/feature_flag/basic_user_list.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class BasicUserList < Grape::Entity + expose :id, documentation: { type: 'integer', example: 1 } + expose :iid, documentation: { type: 'integer', example: 1 } + expose :name, documentation: { type: 'string', example: 'user_list' } + expose :user_xids, documentation: { type: 'string', example: 'user1,user2' } + end + end + end +end diff --git a/lib/api/entities/feature_flag/strategy.rb b/lib/api/entities/feature_flag/strategy.rb index 62178420370..aea38f0e24a 100644 --- a/lib/api/entities/feature_flag/strategy.rb +++ b/lib/api/entities/feature_flag/strategy.rb @@ -8,6 +8,7 @@ module API expose :name, documentation: { type: 'string', example: 'userWithId' } expose :parameters, documentation: { type: 'string', example: '{"userIds": "user1"}' } expose :scopes, using: FeatureFlag::Scope + expose :user_list, using: FeatureFlag::BasicUserList end end end diff --git a/lib/api/entities/feature_flag/user_list.rb b/lib/api/entities/feature_flag/user_list.rb index efb3261658a..47f89cea4d2 100644 --- a/lib/api/entities/feature_flag/user_list.rb +++ b/lib/api/entities/feature_flag/user_list.rb @@ -3,16 +3,12 @@ module API module Entities class FeatureFlag < Grape::Entity - class UserList < Grape::Entity + class UserList < BasicUserList include RequestAwareEntity - expose :id, documentation: { type: 'integer', example: 1 } - expose :iid, documentation: { type: 'integer', example: 1 } expose :project_id, documentation: { type: 'integer', example: 2 } expose :created_at, documentation: { type: 'dateTime', example: '2020-02-04T08:13:10.507Z' } expose :updated_at, documentation: { type: 'dateTime', example: '2020-02-04T08:13:10.507Z' } - expose :name, documentation: { type: 'string', example: 'user_list' } - expose :user_xids, documentation: { type: 'string', example: 'user1,user2' } expose :path do |list| project_feature_flags_user_list_path(list.project, list) diff --git a/lib/api/entities/merge_request_diff.rb b/lib/api/entities/merge_request_diff.rb index 3eda1400855..23a2631a485 100644 --- a/lib/api/entities/merge_request_diff.rb +++ b/lib/api/entities/merge_request_diff.rb @@ -4,7 +4,7 @@ module API module Entities class MergeRequestDiff < Grape::Entity expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha, - :created_at, :merge_request_id, :state, :real_size + :created_at, :merge_request_id, :state, :real_size, :patch_id_sha end end end diff --git a/lib/api/entities/ml/mlflow/get_run.rb b/lib/api/entities/ml/mlflow/get_run.rb new file mode 100644 index 00000000000..4bf10f987cc --- /dev/null +++ b/lib/api/entities/ml/mlflow/get_run.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class GetRun < Grape::Entity + expose :itself, using: Run, as: :run + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/run.rb b/lib/api/entities/ml/mlflow/run.rb index 01d85e8862b..10e2434521d 100644 --- a/lib/api/entities/ml/mlflow/run.rb +++ b/lib/api/entities/ml/mlflow/run.rb @@ -5,13 +5,11 @@ module API module Ml module Mlflow class Run < Grape::Entity - expose :run do - expose :itself, using: RunInfo, as: :info - expose :data do - expose :metrics, using: Metric - expose :params, using: KeyValue - expose :metadata, as: :tags, using: KeyValue - end + expose :itself, using: RunInfo, as: :info + expose :data do + expose :metrics, using: Metric + expose :params, using: KeyValue + expose :metadata, as: :tags, using: KeyValue end end end diff --git a/lib/api/entities/ml/mlflow/search_runs.rb b/lib/api/entities/ml/mlflow/search_runs.rb new file mode 100644 index 00000000000..21c2d58452e --- /dev/null +++ b/lib/api/entities/ml/mlflow/search_runs.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class SearchRuns < Grape::Entity # rubocop:disable Search/NamespacedClass + expose :candidates, with: Run, as: :runs + expose :next_page_token + end + end + end + end +end diff --git a/lib/api/entities/note.rb b/lib/api/entities/note.rb index 6ed5ca43fbb..c693edc611b 100644 --- a/lib/api/entities/note.rb +++ b/lib/api/entities/note.rb @@ -4,7 +4,7 @@ module API module Entities class Note < Grape::Entity # Only Issue and MergeRequest have iid - NOTEABLE_TYPES_WITH_IID = %w(Issue MergeRequest).freeze + NOTEABLE_TYPES_WITH_IID = %w[Issue MergeRequest].freeze expose :id expose :type diff --git a/lib/api/entities/personal_access_token.rb b/lib/api/entities/personal_access_token.rb index 3ec91ca5fc9..b9f831021a1 100644 --- a/lib/api/entities/personal_access_token.rb +++ b/lib/api/entities/personal_access_token.rb @@ -13,7 +13,7 @@ module API expose :active?, as: :active, documentation: { type: 'boolean' } expose :expires_at, documentation: { type: 'dateTime', example: '2020-08-31T15:53:00.073Z' } do |personal_access_token| - personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil + personal_access_token.expires_at ? personal_access_token.expires_at.iso8601 : nil end end end diff --git a/lib/api/entities/project_integration_basic.rb b/lib/api/entities/project_integration_basic.rb index d7e111b990e..ad6128e3498 100644 --- a/lib/api/entities/project_integration_basic.rb +++ b/lib/api/entities/project_integration_basic.rb @@ -29,3 +29,5 @@ module API end end end + +API::Entities::ProjectIntegrationBasic.prepend_mod diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index 1846ddf6833..4ed288ee997 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -63,7 +63,8 @@ module API optional :version, type: String, desc: 'The version of the feature flag. Must be `new_version_flag`. Omit to create a Legacy feature flag.' optional :strategies, type: Array do requires :name, type: String, desc: 'The strategy name. Can be `default`, `gradualRolloutUserId`, `userWithId`, or `gitlabUserList`. In GitLab 13.5 and later, can be `flexibleRollout`' - requires :parameters, type: JSON, desc: 'The strategy parameters as a JSON-formatted string e.g. `{"userIds":"user1"}`', documentation: { type: 'String' } + optional :parameters, type: JSON, desc: 'The strategy parameters as a JSON-formatted string e.g. `{"userIds":"user1"}`', documentation: { type: 'String' } + optional :user_list_id, type: Integer, desc: "The ID of the feature flag user list. If strategy is `gitlabUserList`." optional :scopes, type: Array do requires :environment_scope, type: String, desc: 'The environment scope of the scope' end @@ -131,6 +132,7 @@ module API optional :id, type: Integer, desc: 'The feature flag strategy ID' optional :name, type: String, desc: 'The strategy name' optional :parameters, type: JSON, desc: 'The strategy parameters as a JSON-formatted string e.g. `{"userIds":"user1"}`', documentation: { type: 'String' } + optional :user_list_id, type: Integer, desc: "The ID of the feature flag user list" optional :_destroy, type: Boolean, desc: 'Delete the strategy when true' optional :scopes, type: Array do optional :id, type: Integer, desc: 'The scope id' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index b7f21bd6c22..e967b88e500 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -8,6 +8,7 @@ module API include Helpers::PaginationStrategies include Gitlab::Ci::Artifacts::Logger include Gitlab::Utils::StrongMemoize + include Gitlab::RackLoadBalancingHelpers SUDO_HEADER = "HTTP_SUDO" GITLAB_SHARED_SECRET_HEADER = "Gitlab-Shared-Secret" @@ -91,9 +92,7 @@ module API save_current_token_in_env if @current_user - ::ApplicationRecord - .sticking - .stick_or_unstick_request(env, :user, @current_user.id) + load_balancer_stick_request(::ApplicationRecord, :user, @current_user.id) end @current_user @@ -185,6 +184,30 @@ module API end # rubocop: disable CodeReuse/ActiveRecord + def find_pipeline(id) + return unless id + + if id.to_s =~ INTEGER_ID_REGEX + ::Ci::Pipeline.find_by(id: id) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def find_pipeline!(id) + pipeline = find_pipeline(id) + check_pipeline_access(pipeline) + end + + def check_pipeline_access(pipeline) + return forbidden! unless authorized_project_scope?(pipeline&.project) + + return pipeline if can?(current_user, :read_pipeline, pipeline) + return unauthorized! if authenticate_non_public? + + not_found!('Pipeline') + end + + # rubocop: disable CodeReuse/ActiveRecord def find_group(id) if id.to_s =~ INTEGER_ID_REGEX Group.find_by(id: id) @@ -686,8 +709,8 @@ module API namespace_id: namespace_id, project_id: project_id ) - rescue StandardError => error - Gitlab::AppLogger.warn("Internal Event tracking event failed for event: #{event_name}, message: #{error.message}") + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name) end def order_by_similarity?(allow_unauthorized: true) diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 855648f2ef0..265e9ffcdbd 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -27,7 +27,7 @@ module API options[:route_options][:params].map do |key, val| param_type = val[:type] # Search for parameters with Array types (e.g. "[String]", "[Integer]", etc.) - if param_type =~ %r(\[\w*\]) + if param_type =~ %r{\[\w*\]} key end end.compact.to_set diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 53117af8648..8f846fe7348 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -124,95 +124,6 @@ module API ].freeze end - def self.chat_notification_events - [ - { - required: false, - name: :commit_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for commit_events' - }, - { - required: false, - name: :push_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for push_events' - }, - { - required: false, - name: :issues_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for issues_events' - }, - { - required: false, - name: :incident_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for incident_events' - }, - { - required: false, - name: :alert_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for alert_events' - }, - { - required: false, - name: :confidential_issues_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for confidential_issues_events' - }, - { - required: false, - name: :merge_requests_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for merge_requests_events' - }, - { - required: false, - name: :note_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for note_events' - }, - { - required: false, - name: :confidential_note_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for confidential_note_events' - }, - { - required: false, - name: :tag_push_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for tag_push_events' - }, - { - required: false, - name: :deployment_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for deployment_events' - }, - { - required: false, - name: :job_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for job_events' - }, - { - required: false, - name: :pipeline_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for pipeline_events' - }, - { - required: false, - name: :wiki_page_events, - type: ::Grape::API::Boolean, - desc: 'Enable notifications for wiki_page_events' - } - ].freeze - end - def self.integrations { 'apple-app-store' => [ @@ -453,7 +364,6 @@ module API desc: 'Branches for which notifications are to be sent' }, chat_notification_flags, - chat_notification_events, chat_notification_channels ].flatten, 'drone-ci' => [ @@ -548,8 +458,7 @@ module API name: :branches_to_be_notified, type: String, desc: 'Branches for which notifications are to be sent' - }, - chat_notification_events + } ].flatten, 'harbor' => [ { @@ -813,8 +722,7 @@ module API name: :webhook, type: String, desc: 'The Pumble chat webhook. For example, https://api.pumble.com/workspaces/x/...' - }, - chat_notification_events + } ].flatten, 'pushover' => [ { @@ -919,8 +827,7 @@ module API 'slack' => [ chat_notification_settings, chat_notification_flags, - chat_notification_channels, - chat_notification_events + chat_notification_channels ].flatten, 'microsoft-teams' => [ { @@ -940,8 +847,7 @@ module API 'mattermost' => [ chat_notification_settings, chat_notification_flags, - chat_notification_channels, - chat_notification_events + chat_notification_channels ].flatten, 'teamcity' => [ { @@ -988,7 +894,7 @@ module API type: String, desc: 'Unique identifier for the target chat or username of the target channel (in the format @channelusername)' }, - chat_notification_events + chat_notification_flags ].flatten, 'unify-circuit' => [ { @@ -996,8 +902,7 @@ module API name: :webhook, type: String, desc: 'The Unify Circuit webhook. e.g. https://circuit.com/rest/v2/webhooks/incoming/…' - }, - chat_notification_events + } ].flatten, 'webex-teams' => [ { @@ -1005,8 +910,7 @@ module API name: :webhook, type: String, desc: 'The Webex Teams webhook. For example, https://api.ciscospark.com/v1/webhooks/incoming/...' - }, - chat_notification_events + } ].flatten, 'zentao' => [ { @@ -1082,12 +986,17 @@ module API ::Integrations::PipelinesEmail, ::Integrations::Pivotaltracker, ::Integrations::Prometheus, + ::Integrations::Pumble, ::Integrations::Pushover, ::Integrations::Redmine, + ::Integrations::Shimo, ::Integrations::Slack, ::Integrations::SlackSlashCommands, ::Integrations::SquashTm, ::Integrations::Teamcity, + ::Integrations::Telegram, + ::Integrations::UnifyCircuit, + ::Integrations::WebexTeams, ::Integrations::Youtrack, ::Integrations::Zentao ] diff --git a/lib/api/helpers/kubernetes/agent_helpers.rb b/lib/api/helpers/kubernetes/agent_helpers.rb new file mode 100644 index 00000000000..50a8c2a5aed --- /dev/null +++ b/lib/api/helpers/kubernetes/agent_helpers.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module API + module Helpers + module Kubernetes + module AgentHelpers + include Gitlab::Utils::StrongMemoize + + def authenticate_gitlab_kas_request! + render_api_error!('KAS JWT authentication invalid', 401) unless Gitlab::Kas.verify_api_request(headers) + end + + def agent_token + cluster_agent_token_from_authorization_token + end + strong_memoize_attr :agent_token + + def agent + agent_token.agent + end + strong_memoize_attr :agent + + def gitaly_info(project) + gitaly_features = Feature::Gitaly.server_feature_flags + + Gitlab::GitalyClient.connection_data(project.repository_storage).merge(features: gitaly_features) + end + + def gitaly_repository(project) + project.repository.gitaly_repository.to_h + end + + def check_feature_enabled + not_found!('Internal API not found') unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops) + end + + def check_agent_token + unauthorized! unless agent_token + + ::Clusters::AgentTokens::TrackUsageService.new(agent_token).execute + end + + def agent_has_access_to_project?(project) + Guest.can?(:download_code, project) || agent.has_access_to?(project) + end + + def increment_unique_events + events = params[:unique_counters]&.slice( + :agent_users_using_ci_tunnel, + :k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access, + :k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access, + :k8s_api_proxy_requests_unique_users_via_pat_access, :k8s_api_proxy_requests_unique_agents_via_pat_access, + :flux_git_push_notified_unique_projects + ) + + events&.each do |event, entity_ids| + increment_unique_values(event, entity_ids) + end + end + + def increment_count_events + events = params[:counters]&.slice( + :gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total, + :k8s_api_proxy_requests_via_ci_access, :k8s_api_proxy_requests_via_user_access, + :k8s_api_proxy_requests_via_pat_access + ) + + Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) + end + + def update_configuration(agent:, config:) + ::Clusters::Agents::Authorizations::CiAccess::RefreshService.new(agent, config: config).execute + ::Clusters::Agents::Authorizations::UserAccess::RefreshService.new(agent, config: config).execute + end + + def retrieve_user_from_session_cookie + # Load session + public_session_id_string = + begin + Gitlab::Kas::UserAccess.decrypt_public_session_id(params[:access_key]) + rescue StandardError + bad_request!('Invalid access_key') + end + + session_id = Rack::Session::SessionId.new(public_session_id_string) + session = ActiveSession.sessions_from_ids([session_id.private_id]).first + unauthorized!('Invalid session') unless session + + # CSRF check + unless ::Gitlab::Kas::UserAccess.valid_authenticity_token?(session.symbolize_keys, params[:csrf_token]) + unauthorized!('CSRF token does not match') + end + + # Load user + user = Warden::SessionSerializer.new('rack.session' => session).fetch(:user) + unauthorized!('Invalid user in session') unless user + user + end + + def retrieve_user_from_personal_access_token + return unless access_token.present? + + validate_access_token!(scopes: [Gitlab::Auth::K8S_PROXY_SCOPE]) + + ::PersonalAccessTokens::LastUsedService.new(access_token).execute + + access_token.user || raise(UnauthorizedError) + end + + def access_token + return unless params[:access_key].present? + + PersonalAccessToken.find_by_token(params[:access_key]) + end + strong_memoize_attr :access_token + end + end + end +end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 4b5335840f6..5219c244968 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -102,7 +102,7 @@ module API def finder_params_by_noteable_type_and_id(type, id) target_type = type.name.underscore { target_type: target_type }.tap do |h| - if %w(issue merge_request).include?(target_type) + if %w[issue merge_request].include?(target_type) h[:target_iid] = id else h[:target_id] = id diff --git a/lib/api/helpers/packages/maven.rb b/lib/api/helpers/packages/maven.rb index 694a1ec6436..71d1ba486ed 100644 --- a/lib/api/helpers/packages/maven.rb +++ b/lib/api/helpers/packages/maven.rb @@ -16,6 +16,66 @@ module API desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end + + def extract_format(file_name) + name, _, format = file_name.rpartition('.') + + if %w[md5 sha1].include?(format) + unprocessable_entity! if Gitlab::FIPS.enabled? && format == 'md5' + + [name, format] + else + [file_name, format] + end + end + + def fetch_package(file_name:, project: nil, group: nil) + order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && + params[:path].exclude?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) + + ::Packages::Maven::PackageFinder.new( + current_user, + project || group, + path: params[:path], + order_by_package_file: order_by_package_file + ).execute + end + + def project + nil + end + + def group + nil + end + + def present_carrierwave_file_with_head_support!(package_file, supports_direct_download: true) + package_file.package.touch_last_downloaded_at + file = package_file.file + + if head_request_on_aws_file?(file, supports_direct_download) && !file.file_storage? + return redirect(signed_head_url(file)) + end + + present_carrierwave_file!(file, supports_direct_download: supports_direct_download) + end + + def signed_head_url(file) + fog_storage = ::Fog::Storage.new(file.fog_credentials) + fog_dir = fog_storage.directories.new(key: file.fog_directory) + fog_file = fog_dir.files.new(key: file.path) + expire_at = ::Fog::Time.now + file.fog_authenticated_url_expiration + + fog_file.collection.head_url(fog_file.key, expire_at) + end + + def head_request_on_aws_file?(file, supports_direct_download) + Gitlab.config.packages.object_store.enabled && + supports_direct_download && + file.class.direct_download_enabled? && + request.head? && + file.fog_credentials[:provider] == 'AWS' + end end end end diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index f3b3a299204..6529bb43993 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -118,10 +118,7 @@ module API def track_snowplow_event(action_name, category, args) event_name = "i_package_#{action_name}" key_path = "counts.package_events_i_package_#{action_name}" - service_ping_context = Gitlab::Tracking::ServicePingContext.new( - data_source: :redis, - key_path: key_path - ).to_context + service_ping_context = Gitlab::Usage::MetricDefinition.context_for(key_path).to_context Gitlab::Tracking.event( category, diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 699d3f360d9..8a0ec1c1abf 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -10,9 +10,9 @@ module API params :optional_project_params_ce do optional :description, type: String, desc: 'The description of the project' - optional :build_git_strategy, type: String, values: %w(fetch clone), desc: 'The Git strategy. Defaults to `fetch`' + optional :build_git_strategy, type: String, values: %w[fetch clone], desc: 'The Git strategy. Defaults to `fetch`' optional :build_timeout, type: Integer, desc: 'Build timeout' - optional :auto_cancel_pending_pipelines, type: String, values: %w(disabled enabled), desc: 'Auto-cancel pending pipelines' + optional :auto_cancel_pending_pipelines, type: String, values: %w[disabled enabled], desc: 'Auto-cancel pending pipelines' optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :service_desk_enabled, type: Boolean, desc: 'Disable or enable the service desk' @@ -23,22 +23,22 @@ module API optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' - optional :issues_access_level, type: String, values: %w(disabled private enabled), desc: 'Issues access level. One of `disabled`, `private` or `enabled`' - optional :repository_access_level, type: String, values: %w(disabled private enabled), desc: 'Repository access level. One of `disabled`, `private` or `enabled`' - optional :merge_requests_access_level, type: String, values: %w(disabled private enabled), desc: 'Merge requests access level. One of `disabled`, `private` or `enabled`' - optional :forking_access_level, type: String, values: %w(disabled private enabled), desc: 'Forks access level. One of `disabled`, `private` or `enabled`' - optional :wiki_access_level, type: String, values: %w(disabled private enabled), desc: 'Wiki access level. One of `disabled`, `private` or `enabled`' - optional :builds_access_level, type: String, values: %w(disabled private enabled), desc: 'Builds access level. One of `disabled`, `private` or `enabled`' - optional :snippets_access_level, type: String, values: %w(disabled private enabled), desc: 'Snippets access level. One of `disabled`, `private` or `enabled`' - optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`' - optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' - optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' - optional :security_and_compliance_access_level, type: String, values: %w(disabled private enabled), desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' - optional :releases_access_level, type: String, values: %w(disabled private enabled), desc: 'Releases access level. One of `disabled`, `private` or `enabled`' - optional :environments_access_level, type: String, values: %w(disabled private enabled), desc: 'Environments access level. One of `disabled`, `private` or `enabled`' - optional :feature_flags_access_level, type: String, values: %w(disabled private enabled), desc: 'Feature flags access level. One of `disabled`, `private` or `enabled`' - optional :infrastructure_access_level, type: String, values: %w(disabled private enabled), desc: 'Infrastructure access level. One of `disabled`, `private` or `enabled`' - optional :monitor_access_level, type: String, values: %w(disabled private enabled), desc: 'Monitor access level. One of `disabled`, `private` or `enabled`' + optional :issues_access_level, type: String, values: %w[disabled private enabled], desc: 'Issues access level. One of `disabled`, `private` or `enabled`' + optional :repository_access_level, type: String, values: %w[disabled private enabled], desc: 'Repository access level. One of `disabled`, `private` or `enabled`' + optional :merge_requests_access_level, type: String, values: %w[disabled private enabled], desc: 'Merge requests access level. One of `disabled`, `private` or `enabled`' + optional :forking_access_level, type: String, values: %w[disabled private enabled], desc: 'Forks access level. One of `disabled`, `private` or `enabled`' + optional :wiki_access_level, type: String, values: %w[disabled private enabled], desc: 'Wiki access level. One of `disabled`, `private` or `enabled`' + optional :builds_access_level, type: String, values: %w[disabled private enabled], desc: 'Builds access level. One of `disabled`, `private` or `enabled`' + optional :snippets_access_level, type: String, values: %w[disabled private enabled], desc: 'Snippets access level. One of `disabled`, `private` or `enabled`' + optional :pages_access_level, type: String, values: %w[disabled private enabled public], desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`' + optional :analytics_access_level, type: String, values: %w[disabled private enabled], desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' + optional :container_registry_access_level, type: String, values: %w[disabled private enabled], desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' + optional :security_and_compliance_access_level, type: String, values: %w[disabled private enabled], desc: 'Security and compliance access level. One of `disabled`, `private` or `enabled`' + optional :releases_access_level, type: String, values: %w[disabled private enabled], desc: 'Releases access level. One of `disabled`, `private` or `enabled`' + optional :environments_access_level, type: String, values: %w[disabled private enabled], desc: 'Environments access level. One of `disabled`, `private` or `enabled`' + optional :feature_flags_access_level, type: String, values: %w[disabled private enabled], desc: 'Feature flags access level. One of `disabled`, `private` or `enabled`' + optional :infrastructure_access_level, type: String, values: %w[disabled private enabled], desc: 'Infrastructure access level. One of `disabled`, `private` or `enabled`' + optional :monitor_access_level, type: String, values: %w[disabled private enabled], desc: 'Monitor access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Deprecated: Use emails_enabled instead.' optional :emails_enabled, type: Boolean, desc: 'Enable email notifications' @@ -65,18 +65,18 @@ module API optional :topics, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of topics for a project' optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for project', documentation: { type: 'file' } optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' - optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :merge_method, type: String, values: %w[ff rebase_merge merge], desc: 'The merge method used when merging merge requests' optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' optional :merge_commit_template, type: String, desc: 'Template used to create merge commit message' optional :squash_commit_template, type: String, desc: 'Template used to create squash commit message' optional :issue_branch_template, type: String, desc: 'Template used to create a branch from an issue' optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled' - optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy' + optional :auto_devops_deploy_strategy, type: String, values: %w[continuous manual timed_incremental], desc: 'Auto Deploy strategy' optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled' optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' optional :packages_enabled, type: Boolean, desc: 'Enable project packages feature' - optional :squash_option, type: String, values: %w(never always default_on default_off), desc: 'Squash default for project. One of `never`, `always`, `default_on`, or `default_off`.' + optional :squash_option, type: String, values: %w[never always default_on default_off], desc: 'Squash default for project. One of `never`, `always`, `default_on`, or `default_off`.' optional :mr_default_target_self, type: Boolean, desc: 'Merge requests of this forked project targets itself by default' end diff --git a/lib/api/helpers/search_helpers.rb b/lib/api/helpers/search_helpers.rb index 66321306496..3c1d4dfac49 100644 --- a/lib/api/helpers/search_helpers.rb +++ b/lib/api/helpers/search_helpers.rb @@ -5,21 +5,21 @@ module API module SearchHelpers def self.global_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones snippet_titles users) + %w[projects issues merge_requests milestones snippet_titles users] end def self.group_search_scopes # This is a separate method so that EE can redefine it. - %w(projects issues merge_requests milestones users) + %w[projects issues merge_requests milestones users] end def self.project_search_scopes # This is a separate method so that EE can redefine it. - %w(issues merge_requests milestones notes wiki_blobs commits blobs users) + %w[issues merge_requests milestones notes wiki_blobs commits blobs users] end def self.search_states - %w(all opened closed merged) + %w[all opened closed merged] end end end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index 3ec0a723808..a73e34f54a3 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -31,7 +31,7 @@ module API INTEGRATIONS[integration.to_param.tr("_", "-")] << { required: false, name: event_name.to_sym, - type: String, + type: ::Grape::API::Boolean, desc: IntegrationsHelper.integration_event_description(integration, event_name) } end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 8783a8dd57c..a88c8b69b81 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -4,81 +4,12 @@ module API # Kubernetes Internal API module Internal class Kubernetes < ::API::Base - include Gitlab::Utils::StrongMemoize - before do check_feature_enabled authenticate_gitlab_kas_request! end - helpers do - def authenticate_gitlab_kas_request! - render_api_error!('KAS JWT authentication invalid', 401) unless Gitlab::Kas.verify_api_request(headers) - end - - def agent_token - @agent_token ||= cluster_agent_token_from_authorization_token - end - - def agent - @agent ||= agent_token.agent - end - - def repo_type - Gitlab::GlRepository::PROJECT - end - - def gitaly_info(project) - gitaly_features = Feature::Gitaly.server_feature_flags - - Gitlab::GitalyClient.connection_data(project.repository_storage).merge(features: gitaly_features) - end - - def gitaly_repository(project) - project.repository.gitaly_repository.to_h - end - - def check_feature_enabled - not_found!('Internal API not found') unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops) - end - - def check_agent_token - unauthorized! unless agent_token - - ::Clusters::AgentTokens::TrackUsageService.new(agent_token).execute - end - - def agent_has_access_to_project?(project) - Guest.can?(:download_code, project) || agent.has_access_to?(project) - end - - def increment_unique_events - events = params[:unique_counters]&.slice( - :agent_users_using_ci_tunnel, - :k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access, - :k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access, - :flux_git_push_notified_unique_projects - ) - - events&.each do |event, entity_ids| - increment_unique_values(event, entity_ids) - end - end - - def increment_count_events - events = params[:counters]&.slice( - :gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total, - :k8s_api_proxy_requests_via_ci_access, :k8s_api_proxy_requests_via_user_access - ) - - Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) - end - - def update_configuration(agent:, config:) - ::Clusters::Agents::Authorizations::CiAccess::RefreshService.new(agent, config: config).execute - ::Clusters::Agents::Authorizations::UserAccess::RefreshService.new(agent, config: config).execute - end - end + helpers ::API::Helpers::Kubernetes::AgentHelpers namespace 'internal' do namespace 'kubernetes' do @@ -155,33 +86,23 @@ module API desc 'Authorize a proxy user request' params do requires :agent_id, type: Integer, desc: 'ID of the agent accessed' - requires :access_type, type: String, values: ['session_cookie'], desc: 'The type of the access key being verified.' + requires :access_type, type: String, values: %w[session_cookie personal_access_token], desc: 'The type of access key being verified.' requires :access_key, type: String, desc: 'The authentication secret for the given access type.' given access_type: ->(val) { val == 'session_cookie' } do requires :csrf_token, type: String, allow_blank: false, desc: 'CSRF token that must be checked when access_type is "session_cookie", to ensure the request originates from a GitLab browsing session.' end end post '/', feature_category: :deployment_management do - # Load session - public_session_id_string = - begin - Gitlab::Kas::UserAccess.decrypt_public_session_id(params[:access_key]) - rescue StandardError - bad_request!('Invalid access_key') - end - - session_id = Rack::Session::SessionId.new(public_session_id_string) - session = ActiveSession.sessions_from_ids([session_id.private_id]).first - unauthorized!('Invalid session') unless session - - # CSRF check - unless ::Gitlab::Kas::UserAccess.valid_authenticity_token?(session.symbolize_keys, params[:csrf_token]) - unauthorized!('CSRF token does not match') - end - # Load user - user = Warden::SessionSerializer.new('rack.session' => session).fetch(:user) - unauthorized!('Invalid user in session') unless user + user = if params[:access_type] == 'session_cookie' + retrieve_user_from_session_cookie + elsif params[:access_type] == 'personal_access_token' + u = retrieve_user_from_personal_access_token + bad_request!('PAT authentication is not enabled') unless Feature.enabled?(:k8s_proxy_pat, u) + u + end + + bad_request!('Unable to get user from request data') if user.nil? # Load agent agent = ::Clusters::Agent.find(params[:agent_id]) @@ -205,6 +126,7 @@ module API optional :flux_git_push_notifications_total, type: Integer, desc: 'The count to increment the flux_git_push_notifications_total metrics by' optional :k8s_api_proxy_requests_via_ci_access, type: Integer, desc: 'The count to increment the k8s_api_proxy_requests_via_ci_access metric by' optional :k8s_api_proxy_requests_via_user_access, type: Integer, desc: 'The count to increment the k8s_api_proxy_requests_via_user_access metric by' + optional :k8s_api_proxy_requests_via_pat_access, type: Integer, desc: 'The count to increment the k8s_api_proxy_requests_via_pat_access metric by' end optional :unique_counters, type: Hash do @@ -213,6 +135,8 @@ module API optional :k8s_api_proxy_requests_unique_agents_via_ci_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via `ci_access`' optional :k8s_api_proxy_requests_unique_users_via_user_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via `user_access`' optional :k8s_api_proxy_requests_unique_agents_via_user_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via `user_access`' + optional :k8s_api_proxy_requests_unique_users_via_pat_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via Personal Access Token' + optional :k8s_api_proxy_requests_unique_agents_via_pat_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via Personal Access Token' optional :flux_git_push_notified_unique_projects, type: Array[Integer], desc: 'An array of projects that have been notified to reconcile their Flux workloads' end end diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 5664a3df589..971b76279a1 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -36,16 +36,7 @@ module API virtual_domain = ::Gitlab::Pages::VirtualHostFinder.new(params[:host]).execute no_content! unless virtual_domain - if virtual_domain.cache_key.present? - # Cache context is not added to make it easier to expire the cache with - # Gitlab::Pages::CacheControl - present_cached virtual_domain, - cache_context: nil, - with: Entities::Internal::Pages::VirtualDomain, - expires_in: ::Gitlab::Pages::CacheControl::EXPIRE - else - present virtual_domain, with: Entities::Internal::Pages::VirtualDomain - end + present virtual_domain, with: Entities::Internal::Pages::VirtualDomain end end end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index eccc55ed158..517de98a148 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -34,18 +34,6 @@ module API .exists? end - def extract_format(file_name) - name, _, format = file_name.rpartition('.') - - if %w(md5 sha1).include?(format) - unprocessable_entity! if Gitlab::FIPS.enabled? && format == 'md5' - - [name, format] - else - [file_name, format] - end - end - # The sha verification done by the maven api is between: # - the sha256 set by workhorse helpers # - the sha256 of the sha1 of the uploaded package file @@ -69,46 +57,6 @@ module API format == 'jar' end - def present_carrierwave_file_with_head_support!(package_file, supports_direct_download: true) - package_file.package.touch_last_downloaded_at - file = package_file.file - - if head_request_on_aws_file?(file, supports_direct_download) && !file.file_storage? - return redirect(signed_head_url(file)) - end - - present_carrierwave_file!(file, supports_direct_download: supports_direct_download) - end - - def signed_head_url(file) - fog_storage = ::Fog::Storage.new(file.fog_credentials) - fog_dir = fog_storage.directories.new(key: file.fog_directory) - fog_file = fog_dir.files.new(key: file.path) - expire_at = ::Fog::Time.now + file.fog_authenticated_url_expiration - - fog_file.collection.head_url(fog_file.key, expire_at) - end - - def head_request_on_aws_file?(file, supports_direct_download) - Gitlab.config.packages.object_store.enabled && - supports_direct_download && - file.class.direct_download_enabled? && - request.head? && - file.fog_credentials[:provider] == 'AWS' - end - - def fetch_package(file_name:, project: nil, group: nil) - order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && - !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) - - ::Packages::Maven::PackageFinder.new( - current_user, - project || group, - path: params[:path], - order_by_package_file: order_by_package_file - ).execute - end - def find_and_present_package_file(package, file_name, format, params) project = package&.project package_file = nil diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 03b9ee03b46..1c0b9c56aa7 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -182,7 +182,7 @@ module API merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) options = serializer_options_for(merge_requests).merge(group: user_group) - if !options[:skip_merge_status_recheck] && ::Feature.enabled?(:batched_api_mergeability_checks, user_group) + unless options[:skip_merge_status_recheck] batch_process_mergeability_checks(merge_requests) # NOTE: skipping individual mergeability checks in the presenter diff --git a/lib/api/metadata.rb b/lib/api/metadata.rb index 788d9843c63..e35d22f656a 100644 --- a/lib/api/metadata.rb +++ b/lib/api/metadata.rb @@ -5,7 +5,7 @@ module API helpers ::API::Helpers::GraphqlHelpers include APIGuard - allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } + allow_access_with_scope [:read_user, :ai_features], if: -> (request) { request.get? || request.head? } before { authenticate! } diff --git a/lib/api/ml/mlflow/api_helpers.rb b/lib/api/ml/mlflow/api_helpers.rb index 66689d8e0ca..19ac0dbba1b 100644 --- a/lib/api/ml/mlflow/api_helpers.rb +++ b/lib/api/ml/mlflow/api_helpers.rb @@ -4,6 +4,14 @@ module API module Ml module Mlflow module ApiHelpers + def check_api_read! + not_found! unless can?(current_user, :read_model_experiments, user_project) + end + + def check_api_write! + unauthorized! unless can?(current_user, :write_model_experiments, user_project) + end + def resource_not_found! render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404) end @@ -32,6 +40,37 @@ module API @candidate ||= find_candidate!(params[:run_id]) end + def candidates_order_params(params) + find_params = { + order_by: nil, + order_by_type: nil, + sort: nil + } + + return find_params if params[:order_by].blank? + + order_by_split = params[:order_by].split(' ') + order_by_column_split = order_by_split[0].split('.') + if order_by_column_split.size == 1 + order_by_column = order_by_column_split[0] + order_by_column_type = 'column' + elsif order_by_column_split[0] == 'metrics' + order_by_column = order_by_column_split[1] + order_by_column_type = 'metric' + else + order_by_column = nil + order_by_column_type = nil + end + + order_by_sort = order_by_split[1] + + { + order_by: order_by_column, + order_by_type: order_by_column_type, + sort: order_by_sort + } + end + def find_experiment!(iid, name) experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found! end diff --git a/lib/api/ml/mlflow/entrypoint.rb b/lib/api/ml/mlflow/entrypoint.rb index 7948949dac6..3e0cb723580 100644 --- a/lib/api/ml/mlflow/entrypoint.rb +++ b/lib/api/ml/mlflow/entrypoint.rb @@ -27,7 +27,8 @@ module API authenticate! - not_found! unless can?(current_user, :read_model_experiments, user_project) + check_api_read! + check_api_write! unless request.get? || request.head? end rescue_from ActiveRecord::ActiveRecordError do |e| diff --git a/lib/api/ml/mlflow/runs.rb b/lib/api/ml/mlflow/runs.rb index f737c6bd497..5b6afffaae1 100644 --- a/lib/api/ml/mlflow/runs.rb +++ b/lib/api/ml/mlflow/runs.rb @@ -26,7 +26,7 @@ module API end post 'create', urgency: :low do present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]), - with: Entities::Ml::Mlflow::Run, packages_url: packages_url + with: Entities::Ml::Mlflow::GetRun, packages_url: packages_url end desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do @@ -38,7 +38,47 @@ module API optional :run_uuid, type: String, desc: 'This parameter is ignored' end get 'get', urgency: :low do - present candidate, with: Entities::Ml::Mlflow::Run, packages_url: packages_url + present candidate, with: Entities::Ml::Mlflow::GetRun, packages_url: packages_url + end + + desc 'Searches runs/candidates within a project' do + success Entities::Ml::Mlflow::Run + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#search-runs' \ + 'experiment_ids supports only a single experiment ID.' \ + 'Introduced in GitLab 16.4' + end + params do + requires :experiment_ids, + type: Array, + desc: 'IDs of the experiments to get searches from, relative to the project' + optional :max_results, + type: Integer, + desc: 'Maximum number of runs/candidates to fetch in a page. Default is 200, maximum in 1000', + default: 200 + optional :order_by, + type: String, + desc: 'Order criteria. Can be by a column of the run/candidate (created_at, name) or by a metric if' \ + 'prefixed by `metrics`. Valid examples: `created_at`, `created_at DESC`, `metrics.my_metric DESC`' \ + 'Sorting by candidate parameter or metadata is not supported.', + default: 'created_at DESC' + optional :page_token, + type: String, + desc: 'Token for pagination' + end + get 'search', urgency: :low do + params[:experiment_id] = params[:experiment_ids][0] + + max_results = [params[:max_results], 1000].min + finder_params = candidates_order_params(params) + finder = ::Projects::Ml::CandidateFinder.new(experiment, finder_params) + paginator = finder.execute.keyset_paginate(cursor: params[:page_token], per_page: max_results) + + result = { + candidates: paginator.records, + next_page_token: paginator.cursor_for_next_page + } + + present result, with: Entities::Ml::Mlflow::SearchRuns, packages_url: packages_url end desc 'Updates a Run.' do diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 8cd72d2ab15..e3d292eb2b0 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -39,8 +39,12 @@ module API notification_setting.transaction do new_notification_email = params.delete(:notification_email) - if new_notification_email - ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute + Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.temporary_ignore_tables_in_transaction( + %w[users user_details], url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/424289' + ) do + if new_notification_email + ::Users::UpdateService.new(current_user, user: current_user, notification_email: new_notification_email).execute + end end notification_setting.update(declared_params(include_missing: false)) diff --git a/lib/api/npm_group_packages.rb b/lib/api/npm_group_packages.rb index 1aa3135b186..f2b8e1840a1 100644 --- a/lib/api/npm_group_packages.rb +++ b/lib/api/npm_group_packages.rb @@ -11,6 +11,10 @@ module API def endpoint_scope :group end + + def group_or_namespace + group + end end params do diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb index ea92818e76c..1805edceb2c 100644 --- a/lib/api/npm_instance_packages.rb +++ b/lib/api/npm_instance_packages.rb @@ -10,6 +10,10 @@ module API def endpoint_scope :instance end + + def group_or_namespace + top_namespace_from(params[:package_name]) + end end namespace 'packages/npm' do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index bff645700f5..dbc789c68b6 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -84,6 +84,8 @@ module API file_name: file_name(symbol_package) ) + check_duplicate(file_params, symbol_package) + package = ::Packages::CreateTemporaryPackageService.new( project, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:nuget, name: temp_file_name(symbol_package)) @@ -98,6 +100,14 @@ module API created! end + def check_duplicate(file_params, symbol_package) + return if symbol_package || Feature.disabled?(:nuget_duplicates_option, project_or_group.namespace) + + service_params = file_params.merge(remote_url: params['package.remote_url']) + response = ::Packages::Nuget::CheckDuplicatesService.new(project_or_group, current_user, service_params).execute + render_api_error!(response.message, response.reason) if response.error? + end + def publish_package(symbol_package: false) upload_nuget_package_file(symbol_package: symbol_package) do |package| track_package_event( @@ -119,9 +129,25 @@ module API end def format_filename(package) - return "#{params[:package_filename]}.#{params[:format]}" if Feature.disabled?(:nuget_normalized_version, project_or_group) || package.version == params[:package_version] + return "#{params[:package_filename]}.#{params[:format]}" if package.version == params[:package_version] return "#{params[:package_filename].sub(params[:package_version], package.version)}.#{params[:format]}" if package.normalized_nuget_version == params[:package_version] end + + def present_odata_entry + project = find_project(params[:project_id]) + + not_found! unless project + + env['api.format'] = :binary + content_type 'application/xml; charset=utf-8' + + odata_entry = ::Packages::Nuget::OdataPackageEntryService + .new(project, declared_params) + .execute + .payload + + present odata_entry + end end params do @@ -317,5 +343,74 @@ module API end end end + + params do + requires :project_id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project', + regexp: ::API::Concerns::Packages::Nuget::PrivateEndpoints::POSITIVE_INTEGER_REGEX + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':project_id/packages/nuget/v2' do + # https://joelverhagen.github.io/NuGetUndocs/?http#endpoint-find-packages-by-id + desc 'The NuGet V2 Feed Find Packages by ID endpoint' do + detail 'This feature was introduced in GitLab 16.4' + success code: 200 + failure [ + { code: 404, message: 'Not Found' }, + { code: 400, message: 'Bad Request' } + ] + tags %w[nuget_packages] + end + + params do + requires :id, as: :package_name, type: String, allow_blank: false, coerce_with: ->(val) { val.delete("'") }, + desc: 'The NuGet package name', regexp: Gitlab::Regex.nuget_package_name_regex, + documentation: { example: 'mynugetpkg' } + end + get 'FindPackagesById\(\)', urgency: :low do + present_odata_entry + end + + # https://joelverhagen.github.io/NuGetUndocs/?http#endpoint-enumerate-packages + desc 'The NuGet V2 Feed Enumerate Packages endpoint' do + detail 'This feature was introduced in GitLab 16.4' + success code: 200 + failure [ + { code: 404, message: 'Not Found' }, + { code: 400, message: 'Bad Request' } + ] + tags %w[nuget_packages] + end + + params do + requires :$filter, as: :package_name, type: String, allow_blank: false, + coerce_with: ->(val) { val.match(/tolower\(Id\) eq '(.+?)'/)&.captures&.first }, + desc: 'The NuGet package name', regexp: Gitlab::Regex.nuget_package_name_regex, + documentation: { example: 'mynugetpkg' } + end + get 'Packages\(\)', urgency: :low do + present_odata_entry + end + + # https://joelverhagen.github.io/NuGetUndocs/?http#endpoint-get-a-single-package + desc 'The NuGet V2 Feed Single Package Metadata endpoint' do + detail 'This feature was introduced in GitLab 16.4' + success code: 200 + failure [ + { code: 404, message: 'Not Found' }, + { code: 400, message: 'Bad Request' } + ] + tags %w[nuget_packages] + end + params do + requires :package_name, type: String, allow_blank: false, desc: 'The NuGet package name', + regexp: Gitlab::Regex.nuget_package_name_regex, documentation: { example: 'mynugetpkg' } + requires :package_version, type: String, allow_blank: false, desc: 'The NuGet package version', + regexp: Gitlab::Regex.nuget_version_regex, documentation: { example: '1.3.0.17' } + end + get 'Packages\(Id=\'*package_name\',Version=\'*package_version\'\)', urgency: :low do + present_odata_entry + end + end + end end end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 8a72ec051dc..7467b8e564e 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -154,7 +154,7 @@ module API { code: 503, message: 'Service unavailable' } ] tags ['project_export'] - produces %w[application/octet-stream application/json] + produces %w[application/octet-stream application/gzip application/json] end params do requires :relation, type: String, project_portable: true, desc: 'Project relation name' diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index c28d0ae2def..60dfc925269 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -10,7 +10,7 @@ module API feature_category :importers urgency :low - before { authenticate! unless route.settings[:skip_authentication] } + before { authenticate! } helpers do def import_params @@ -132,7 +132,6 @@ module API ] tags ['project_import'] end - route_setting :skip_authentication, true get ':id/import' do present user_project, with: Entities::ProjectImportStatus, current_user: current_user end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4131f41743f..98316bf1d4b 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -112,7 +112,7 @@ module API optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' use :pagination - optional :pagination, type: String, values: %w(legacy keyset none), default: 'legacy', desc: 'Specify the pagination method ("none" is only valid if "recursive" is true)' + optional :pagination, type: String, values: %w[legacy keyset none], default: 'legacy', desc: 'Specify the pagination method ("none" is only valid if "recursive" is true)' given pagination: ->(value) { value == 'keyset' } do optional :page_token, type: String, diff --git a/lib/api/search.rb b/lib/api/search.rb index b14fce13f5e..5f78979ec8a 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -7,7 +7,8 @@ module API before do authenticate! - check_rate_limit!(:search_rate_limit, scope: [current_user]) + check_rate_limit!(:search_rate_limit, scope: [current_user], + users_allowlist: Gitlab::CurrentSettings.current_application_settings.search_rate_limit_allowlist) end feature_category :global_search @@ -102,7 +103,7 @@ module API end def snippets? - %w(snippet_titles).include?(params[:scope]).to_s + %w[snippet_titles].include?(params[:scope]).to_s end def entity diff --git a/lib/api/settings.rb b/lib/api/settings.rb index e2dc78fe84a..9616efbfe37 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -45,6 +45,7 @@ module API optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :asset_proxy_allowlist instead. Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' optional :asset_proxy_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :decompress_archive_file_timeout, type: Integer, desc: 'Default timeout for decompressing archived files, in seconds. Set to 0 to disable timeouts.' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_ci_config_path, type: String, desc: 'The instance default CI/CD configuration file and path for new projects' optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' @@ -109,7 +110,6 @@ module API optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, values: %w[github bitbucket bitbucket_server fogbugz git gitlab_project gitea manifest], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' - optional :in_product_marketing_emails_enabled, type: Boolean, desc: 'By default, in-product marketing emails are enabled. To disable these emails, disable this option.' optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.' optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' diff --git a/lib/api/users.rb b/lib/api/users.rb index fff0e9fee06..a01ace3a9c3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -141,7 +141,11 @@ module API users = users.preload(:user_detail) - present paginate(users), options + if Feature.enabled?(:api_keyset_pagination_multi_order) + present paginate_with_strategies(users), options + else + present paginate(users), options + end end # rubocop: enable CodeReuse/ActiveRecord @@ -1021,7 +1025,7 @@ module API # Enabling /user endpoint for the v3 version to allow oauth # authentication through this endpoint. - version %w(v3 v4), using: :path do + version %w[v3 v4], using: :path do desc 'Get the currently authenticated user' do success Entities::UserPublic end diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb index 67dc084cc12..77d76c98e00 100644 --- a/lib/api/validations/validators/bulk_imports.rb +++ b/lib/api/validations/validators/bulk_imports.rb @@ -36,7 +36,8 @@ module API raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], - message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message + message: "must be a relative path and not include protocol, sub-domain, or domain information. " \ + "For example, 'destination/full/path' not 'https://example.com/destination/full/path'" ) end end @@ -51,7 +52,7 @@ module API raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], message: "must be a relative path and not include protocol, sub-domain, or domain information. " \ - "For example, 'source/full/path' not 'https://example.com/source/full/path'" \ + "For example, 'source/full/path' not 'https://example.com/source/full/path'" ) end end |