diff options
Diffstat (limited to 'lib')
220 files changed, 3512 insertions, 1375 deletions
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb index 3277acc1b52..a443f7f3476 100644 --- a/lib/api/admin/ci/variables.rb +++ b/lib/api/admin/ci/variables.rb @@ -54,6 +54,10 @@ module API type: String, desc: 'The key of the variable. Max 255 characters' + optional :description, + type: String, + desc: 'The description of the variable' + requires :value, type: String, desc: 'The value of a variable' @@ -98,6 +102,10 @@ module API type: String, desc: 'The key of a variable' + optional :description, + type: String, + desc: 'The description of the variable' + optional :value, type: String, desc: 'The value of a variable' diff --git a/lib/api/api.rb b/lib/api/api.rb index 97e09795f49..94b433193dd 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -330,7 +330,7 @@ module API mount ::API::Suggestions mount ::API::SystemHooks mount ::API::Tags - mount ::API::Terraform::Modules::V1::Packages + mount ::API::Terraform::Modules::V1::NamespacePackages mount ::API::Terraform::Modules::V1::ProjectPackages mount ::API::Terraform::State mount ::API::Terraform::StateVersion diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index b5123ab49dc..f369fc5e183 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -183,7 +183,7 @@ module API .new(current_user: current_user, pipeline: pipeline, params: params) .execute - builds = builds.with_preloads + builds = builds.with_preloads.preload(:metadata) # rubocop:disable CodeReuse/ActiveRecord -- preload job.archived? present paginate(builds), with: Entities::Ci::Job end diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 17bee275c51..300c30faf4a 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -94,6 +94,14 @@ module API forbidden!("No access granted") unless can?(current_user, :read_builds, runner) end + + def preload_job_associations(jobs) + jobs.preload( # rubocop: disable CodeReuse/ActiveRecord -- this preload is tightly related to the endpoint + :user, + { pipeline: { project: [:route, { namespace: :route }] } }, + { project: [:route, { namespace: :route }] } + ) + end end resource :runners do @@ -217,25 +225,24 @@ module API end params do requires :id, type: Integer, desc: 'The ID of a runner' + optional :system_id, type: String, desc: 'System ID associated with the runner manager' optional :status, type: String, desc: 'Status of the job', values: ::Ci::Build::AVAILABLE_STATUSES optional :order_by, type: String, desc: 'Order by `id`', values: ::Ci::RunnerJobsFinder::ALLOWED_INDEXED_COLUMNS optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort by `asc` or `desc` order. ' \ - 'Specify `order_by` as well, including for `id`' + 'Specify `order_by` as well, including for `id`' + optional :cursor, type: String, desc: 'Cursor for obtaining the next set of records' use :pagination end get ':id/jobs' do runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) + # Optimize query when filtering by runner managers by not asking for count + paginator_params = params[:pagination] == :keyset || params[:system_id].blank? ? {} : { without_count: true } + jobs = ::Ci::RunnerJobsFinder.new(runner, current_user, params).execute - jobs = jobs.preload( # rubocop: disable CodeReuse/ActiveRecord - [ - :user, - { pipeline: { project: [:route, { namespace: :route }] } }, - { project: [:route, { namespace: :route }] } - ] - ) - jobs = paginate(jobs) + jobs = preload_job_associations(jobs) + jobs = paginate_with_strategies(jobs, paginator_params: paginator_params) jobs.each(&:commit) # batch loads all commits in the page present jobs, with: Entities::Ci::JobBasicWithProject diff --git a/lib/api/draft_notes.rb b/lib/api/draft_notes.rb index 6fadc68233d..3d046ec4a9b 100644 --- a/lib/api/draft_notes.rb +++ b/lib/api/draft_notes.rb @@ -22,7 +22,9 @@ module API end def delete_draft_note(draft_note) - ::DraftNotes::DestroyService.new(user_project, current_user).execute(draft_note) + ::DraftNotes::DestroyService + .new(merge_request(params: params), current_user) + .execute(draft_note) end def publish_draft_note(params:) diff --git a/lib/api/entities/bulk_imports/entity_failure.rb b/lib/api/entities/bulk_imports/entity_failure.rb index 08708a7c961..4771e6cb894 100644 --- a/lib/api/entities/bulk_imports/entity_failure.rb +++ b/lib/api/entities/bulk_imports/entity_failure.rb @@ -6,7 +6,7 @@ module API class EntityFailure < Grape::Entity expose :relation, documentation: { type: 'string', example: 'label' } expose :exception_message, documentation: { type: 'string', example: 'error message' } do |failure| - ::Projects::ImportErrorFilter.filter_message(failure.exception_message.truncate(72)) + ::Projects::ImportErrorFilter.filter_message(failure.exception_message).truncate(255) end expose :exception_class, documentation: { type: 'string', example: 'Exception' } expose :correlation_id_value, documentation: { type: 'string', example: 'dfcf583058ed4508e4c7c617bd7f0edd' } diff --git a/lib/api/entities/ci/job.rb b/lib/api/entities/ci/job.rb index d9e6b7eed75..2f748d28abf 100644 --- a/lib/api/entities/ci/job.rb +++ b/lib/api/entities/ci/job.rb @@ -12,6 +12,7 @@ module API expose :runner, with: ::API::Entities::Ci::Runner expose :artifacts_expire_at, documentation: { type: 'dateTime', example: '2016-01-19T09:05:50.355Z' } + expose :archived?, as: :archived, documentation: { type: 'boolean', example: false } expose( :tag_list, diff --git a/lib/api/entities/diff.rb b/lib/api/entities/diff.rb index cc53736a5b1..e1e6ce26263 100644 --- a/lib/api/entities/diff.rb +++ b/lib/api/entities/diff.rb @@ -16,6 +16,7 @@ module API expose :new_file?, as: :new_file, documentation: { type: 'boolean' } expose :renamed_file?, as: :renamed_file, documentation: { type: 'boolean' } expose :deleted_file?, as: :deleted_file, documentation: { type: 'boolean' } + expose :generated?, as: :generated_file, documentation: { type: 'boolean' } end end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index 1a1765c2e0a..14491c2396a 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -23,6 +23,7 @@ module API expose :full_name, :full_path expose :created_at expose :parent_id + expose :organization_id expose :shared_runners_setting expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index 56519e2bf08..39bb54bfc5a 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -97,6 +97,11 @@ module API expose :squash expose :squash_on_merge?, as: :squash_on_merge expose :task_completion_status + + # #cannot_be_merged? is generally indicative of conflicts, and is set via + # MergeRequests::MergeabilityCheckService. However, it can also indicate + # that either #has_no_commits? or #branch_missing? are true. + # expose :cannot_be_merged?, as: :has_conflicts expose :mergeable_discussions_state?, as: :blocking_discussions_resolved diff --git a/lib/api/entities/ml/mlflow/model_version.rb b/lib/api/entities/ml/mlflow/model_version.rb index 10fdf3822a5..d57def4e1f2 100644 --- a/lib/api/entities/ml/mlflow/model_version.rb +++ b/lib/api/entities/ml/mlflow/model_version.rb @@ -18,7 +18,7 @@ module API expose :run_id expose :status expose :status_message - expose :metadata + expose :metadata, as: :tags, using: KeyValue expose :run_link expose :aliases, documentation: { is_array: true, type: String } @@ -68,10 +68,6 @@ module API "" end - def metadata - [] - end - def run_link "" end diff --git a/lib/api/entities/ml/mlflow/search_experiments.rb b/lib/api/entities/ml/mlflow/search_experiments.rb new file mode 100644 index 00000000000..9673cb1b6fd --- /dev/null +++ b/lib/api/entities/ml/mlflow/search_experiments.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class SearchExperiments < Grape::Entity # rubocop:disable Search/NamespacedClass -- Not related to search + expose :experiments, with: Experiment + expose :next_page_token + end + end + end + end +end diff --git a/lib/api/entities/pages/deployments.rb b/lib/api/entities/pages/deployments.rb new file mode 100644 index 00000000000..143fbe93344 --- /dev/null +++ b/lib/api/entities/pages/deployments.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Pages + class Deployments < Grape::Entity + expose :created_at + expose :url + expose :path_prefix + expose :root_directory + end + end + end +end diff --git a/lib/api/entities/pages/project_settings.rb b/lib/api/entities/pages/project_settings.rb new file mode 100644 index 00000000000..81a48fe8bd3 --- /dev/null +++ b/lib/api/entities/pages/project_settings.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Pages + class ProjectSettings < Grape::Entity + expose :url + expose :deployments, using: "API::Entities::Pages::Deployments" + expose :unique_domain_enabled?, as: :is_unique_domain_enabled + expose :force_https?, as: :force_https + end + end + end +end diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb index e04ddd52f29..3824e3f1b37 100644 --- a/lib/api/entities/user_preferences.rb +++ b/lib/api/entities/user_preferences.rb @@ -8,5 +8,3 @@ module API end end end - -API::Entities::UserPreferences.prepend_mod_with('API::Entities::UserPreferences', with_descendants: true) diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index f320fa06394..94702c36c85 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -106,11 +106,31 @@ module API declared_params(include_missing: false) ) - variable = ::Ci::ChangeVariableService.new( - container: user_group, - current_user: current_user, - params: { action: :update, variable_params: filtered_params } - ).execute + # If the 'filter' parameter is provided, the user is updating a scoped variable + # and we need to use `find_variable` to make sure we update the correct one. + # However, this would result in an error response in case the user attempts to + # update a scoped variable without providing a filter. This error response is + # technically correct, because updating a scoped variable without specifying + # the targeted scope causes non-deterministic behavior. But this endpoint was + # originally introduced without the ability to specify a filter, and returning + # an error in these cases now would be considered a breaking change. + # Thus we only use the new/correct code if the user provided a filter. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136475 + if params.key?('filter') + variable = find_variable(user_group, params) + + variable = ::Ci::ChangeVariableService.new( + container: user_group, + current_user: current_user, + params: { action: :update, variable: variable, variable_params: filtered_params } + ).execute + else + variable = ::Ci::ChangeVariableService.new( + container: user_group, + current_user: current_user, + params: { action: :update, variable_params: filtered_params } + ).execute + end if variable.valid? present variable, with: Entities::Ci::Variable diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 1ff64cd2ffd..7b755a76f29 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -213,11 +213,15 @@ module API requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + optional :organization_id, type: Integer, desc: 'The organization id for the group' use :optional_params end post feature_category: :groups_and_projects, urgency: :low do - parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? + organization = find_organization!(params[:organization_id]) if params[:organization_id].present? + authorize! :create_group, organization if organization + + parent_group = find_group!(params[:parent_id], organization: organization) if params[:parent_id].present? if parent_group authorize! :create_subgroup, parent_group else diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index f5dcbc07704..a59734d643d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -184,8 +184,7 @@ module API return true unless job_token_authentication? return true unless route_authentication_setting[:job_token_scope] == :project - ::Feature.enabled?(:ci_job_token_scope, project) && - current_authenticated_job.project == project + current_authenticated_job.project == project end # rubocop: disable CodeReuse/ActiveRecord @@ -212,18 +211,25 @@ module API not_found!('Pipeline') end + def find_organization!(id) + organization = Organizations::Organization.find_by_id(id) + check_organization_access(organization) + end + # rubocop: disable CodeReuse/ActiveRecord - def find_group(id) + def find_group(id, organization: nil) + collection = organization.present? ? Group.in_organization(organization) : Group.all + if id.to_s =~ INTEGER_ID_REGEX - Group.find_by(id: id) + collection.find_by(id: id) else - Group.find_by_full_path(id) + collection.find_by_full_path(id) end end # rubocop: enable CodeReuse/ActiveRecord - def find_group!(id) - group = find_group(id) + def find_group!(id, organization: nil) + group = find_group(id, organization: organization) check_group_access(group) end @@ -836,6 +842,12 @@ module API @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] end + def check_organization_access(organization) + return organization if can?(current_user, :read_organization, organization) + + not_found!('Organization') + end + def secret_token Gitlab::Shell.secret_token end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index dd3009ff1d7..b450718a7d0 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -7,35 +7,6 @@ module API # The data structures inside this model are returned using class methods, # allowing EE to extend them where necessary. module IntegrationsHelpers - def self.chat_notification_settings - [ - { - required: true, - name: :webhook, - type: String, - desc: 'The chat webhook' - }, - { - required: false, - name: :username, - type: String, - desc: 'The chat username' - }, - { - required: false, - name: :channel, - type: String, - desc: 'The default chat channel' - }, - { - required: false, - name: :branches_to_be_notified, - type: String, - desc: 'Branches for which notifications are to be sent' - } - ].freeze - end - def self.chat_notification_flags [ { @@ -129,58 +100,8 @@ module API 'apple-app-store' => ::Integrations::AppleAppStore.api_fields, 'asana' => ::Integrations::Asana.api_fields, 'assembla' => ::Integrations::Assembla.api_fields, - 'bamboo' => [ - { - required: true, - name: :bamboo_url, - type: String, - desc: 'Bamboo root URL like https://bamboo.example.com' - }, - { - required: false, - name: :enable_ssl_verification, - type: ::Grape::API::Boolean, - desc: 'Enable SSL verification' - }, - { - required: true, - name: :build_key, - type: String, - desc: 'Bamboo build plan key like' - }, - { - required: true, - name: :username, - type: String, - desc: 'A user with API access, if applicable' - }, - { - required: true, - name: :password, - type: String, - desc: 'Password of the user' - } - ], - 'bugzilla' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'New issue URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'Issues URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'Project URL' - } - ], + 'bamboo' => ::Integrations::Bamboo.api_fields, + 'bugzilla' => ::Integrations::Bugzilla.api_fields, 'buildkite' => [ { required: true, @@ -201,54 +122,9 @@ module API desc: 'DEPRECATED: This parameter has no effect since SSL verification will always be enabled' } ], - 'campfire' => [ - { - required: true, - name: :token, - type: String, - desc: 'Campfire token' - }, - { - required: false, - name: :subdomain, - type: String, - desc: 'Campfire subdomain' - }, - { - required: false, - name: :room, - type: String, - desc: 'Campfire room' - } - ], - 'confluence' => [ - { - required: true, - name: :confluence_url, - type: String, - desc: 'The URL of the Confluence Cloud Workspace hosted on atlassian.net' - } - ], - 'custom-issue-tracker' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'New issue URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'Issues URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'Project URL' - } - ], + 'campfire' => ::Integrations::Campfire.api_fields, + 'confluence' => ::Integrations::Confluence.api_fields, + 'custom-issue-tracker' => ::Integrations::CustomIssueTracker.api_fields, 'datadog' => [ { required: true, @@ -293,19 +169,9 @@ module API desc: 'Custom tags in Datadog. Specify one tag per line in the format: "key:value\nkey2:value2"' } ], + 'diffblue-cover' => ::Integrations::DiffblueCover.api_fields, 'discord' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'Discord webhook. For example, https://discord.com/api/webhooks/…' - }, - { - required: false, - name: :branches_to_be_notified, - type: String, - desc: 'Branches for which notifications are to be sent' - }, + ::Integrations::Discord.api_fields, chat_notification_flags, chat_notification_channels ].flatten, @@ -355,40 +221,8 @@ module API desc: 'Branches for which notifications are to be sent' } ], - 'external-wiki' => [ - { - required: true, - name: :external_wiki_url, - type: String, - desc: 'The URL of the external wiki' - } - ], - 'google-play' => [ - { - required: true, - name: :package_name, - type: String, - desc: 'The package name of the app in Google Play' - }, - { - required: true, - name: :service_account_key, - type: String, - desc: 'The Google Play service account key' - }, - { - required: true, - name: :service_account_key_file_name, - type: String, - desc: 'The filename of the Google Play service account key' - }, - { - required: false, - name: :google_play_protected_refs, - type: ::Grape::API::Boolean, - desc: 'Only enable for protected refs' - } - ], + 'external-wiki' => ::Integrations::ExternalWiki.api_fields, + 'google-play' => ::Integrations::GooglePlay.api_fields, 'hangouts-chat' => [ { required: true, @@ -403,32 +237,7 @@ module API desc: 'Branches for which notifications are to be sent' } ].flatten, - 'harbor' => [ - { - required: true, - name: :url, - type: String, - desc: 'The base URL to the Harbor instance which is being linked to this GitLab project. For example, https://demo.goharbor.io.' - }, - { - required: true, - name: :project_name, - type: String, - desc: 'The Project name to the Harbor instance. For example, testproject.' - }, - { - required: true, - name: :username, - type: String, - desc: 'The username created from Harbor interface.' - }, - { - required: true, - name: :password, - type: String, - desc: 'The password of the user.' - } - ], + 'harbor' => ::Integrations::Harbor.api_fields, 'irker' => [ { required: true, @@ -555,14 +364,7 @@ module API desc: 'Enable comments inside Jira issues on each GitLab event (commit / merge request)' } ], - 'mattermost-slash-commands' => [ - { - required: true, - name: :token, - type: String, - desc: 'The Mattermost token' - } - ], + 'mattermost-slash-commands' => ::Integrations::MattermostSlashCommands.api_fields, 'slack-slash-commands' => [ { required: true, @@ -697,77 +499,12 @@ module API desc: 'The sound of the notification' } ], - 'redmine' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'The new issue URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'The project URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'The issues URL' - } - ], - 'ewm' => [ - { - required: true, - name: :new_issue_url, - type: String, - desc: 'New Issue URL' - }, - { - required: true, - name: :project_url, - type: String, - desc: 'Project URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'Issues URL' - } - ], - 'youtrack' => [ - { - required: true, - name: :project_url, - type: String, - desc: 'The project URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'The issues URL' - } - ], - 'clickup' => [ - { - required: true, - name: :project_url, - type: String, - desc: 'The project URL' - }, - { - required: true, - name: :issues_url, - type: String, - desc: 'The issues URL' - } - ], + 'redmine' => ::Integrations::Redmine.api_fields, + 'ewm' => ::Integrations::Ewm.api_fields, + 'youtrack' => ::Integrations::Youtrack.api_fields, + 'clickup' => ::Integrations::Clickup.api_fields, 'slack' => [ - chat_notification_settings, - chat_notification_flags, + ::Integrations::Slack.api_fields, chat_notification_channels ].flatten, 'microsoft-teams' => [ @@ -786,8 +523,7 @@ module API chat_notification_flags ].flatten, 'mattermost' => [ - chat_notification_settings, - chat_notification_flags, + ::Integrations::Mattermost.api_fields, chat_notification_channels ].flatten, 'teamcity' => [ @@ -879,20 +615,7 @@ module API desc: 'The product ID of ZenTao project' } ], - 'squash-tm' => [ - { - required: true, - name: :url, - type: String, - desc: 'The Squash TM webhook URL' - }, - { - required: false, - name: :token, - type: String, - desc: 'The secret token' - } - ] + 'squash-tm' => ::Integrations::SquashTm.api_fields } end @@ -909,6 +632,7 @@ module API ::Integrations::Confluence, ::Integrations::CustomIssueTracker, ::Integrations::Datadog, + ::Integrations::DiffblueCover, ::Integrations::Discord, ::Integrations::DroneCi, ::Integrations::EmailsOnPush, diff --git a/lib/api/helpers/kubernetes/agent_helpers.rb b/lib/api/helpers/kubernetes/agent_helpers.rb index eca26c023cf..18f47fa9955 100644 --- a/lib/api/helpers/kubernetes/agent_helpers.rb +++ b/lib/api/helpers/kubernetes/agent_helpers.rb @@ -40,7 +40,6 @@ module API def increment_unique_events events = params[:unique_counters]&.slice( - :agent_users_using_ci_tunnel, :k8s_api_proxy_requests_unique_agents_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_user_access, :k8s_api_proxy_requests_unique_agents_via_pat_access, @@ -60,6 +59,8 @@ module API ) return if event_lists.blank? + event_lists[:agent_users_using_ci_tunnel] = event_lists.values.flatten + users, projects = load_users_and_projects(event_lists) event_lists.each do |event_name, events| track_events_for(event_name, events, users, projects) diff --git a/lib/api/helpers/packages/maven.rb b/lib/api/helpers/packages/maven.rb index 6c50f4c00a1..f7c8da3e641 100644 --- a/lib/api/helpers/packages/maven.rb +++ b/lib/api/helpers/packages/maven.rb @@ -19,11 +19,11 @@ module API documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end - def extract_format(file_name) + def extract_format(file_name, skip_fips_check: false) name, _, format = file_name.rpartition('.') if %w[md5 sha1].include?(format) - unprocessable_entity! if Gitlab::FIPS.enabled? && format == 'md5' + unprocessable_entity! if !skip_fips_check && Gitlab::FIPS.enabled? && format == 'md5' [name, format] else diff --git a/lib/api/helpers/user_preferences_helpers.rb b/lib/api/helpers/user_preferences_helpers.rb deleted file mode 100644 index 846ad354156..00000000000 --- a/lib/api/helpers/user_preferences_helpers.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module UserPreferencesHelpers - extend ActiveSupport::Concern - extend Grape::API::Helpers - - def update_user_namespace_settings(attrs) - # This method will be redefined in EE. - attrs - end - end - end -end - -API::Helpers::UserPreferencesHelpers.prepend_mod_with('API::Helpers::UserPreferencesHelpers') diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index d3a4d94f8ca..286192f8093 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -127,7 +127,6 @@ module API end optional :unique_counters, type: Hash do - optional :agent_users_using_ci_tunnel, type: Array[Integer], desc: 'An array of user ids that have interacted with CI Tunnel' optional :k8s_api_proxy_requests_unique_users_via_ci_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via `ci_access`' optional :k8s_api_proxy_requests_unique_agents_via_ci_access, type: Array[Integer], desc: 'An array of agents that have interacted with the CI tunnel via `ci_access`' optional :k8s_api_proxy_requests_unique_users_via_user_access, type: Array[Integer], desc: 'An array of users that have interacted with the CI tunnel via `user_access`' diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 14c3fccee32..e969935383a 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -260,7 +260,13 @@ module API authorize_upload! bad_request!('File is too large') if user_project.actual_limits.exceeded?(:maven_max_file_size, params[:file].size) - file_name, format = extract_format(params[:file_name]) + # In FIPS mode, we've already told Workhorse not to generate a + # MD5 checksum via UploadHashFunctions, and the FIPS check above + # ensures that Workhorse obeys that. However, Gradle will attempt to issue a PUT request + # with the MD5 checksum, and the publish step will fail if this endpoint returns a + # 422 (https://github.com/gradle/gradle/blob/v8.5.0/platforms/software/maven/src/main/java/org/gradle/api/publish/maven/internal/publisher/AbstractMavenPublisher.java#L240), + # so we need to skip the second FIPS check here. + file_name, format = extract_format(params[:file_name], skip_fips_check: true) ::Gitlab::Database::LoadBalancing::Session.current.use_primary do result = ::Packages::Maven::FindOrCreatePackageService diff --git a/lib/api/members.rb b/lib/api/members.rb index 56a15c41e1c..908733d4aa1 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -176,7 +176,7 @@ module API source = find_source(source_type, params[:id]) member = source_members(source).find_by!(user_id: params[:user_id]) - check_rate_limit!(:member_delete, scope: [source, current_user]) + check_rate_limit!(:members_delete, scope: [source, current_user]) destroy_conditionally!(member) do ::Members::DestroyService.new(current_user).execute(member, skip_subresources: params[:skip_subresources], unassign_issuables: params[:unassign_issuables]) diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index d0c9400039a..23d7dafdc0a 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -104,7 +104,8 @@ module API put 'reset_approvals', urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) - unauthorized! unless current_user.can?(:reset_merge_request_approvals, merge_request) + unauthorized! unless current_user.can?(:reset_merge_request_approvals, merge_request) && + !merge_request.merged? merge_request.approvals.delete_all diff --git a/lib/api/ml/mlflow/api_helpers.rb b/lib/api/ml/mlflow/api_helpers.rb index 66d79753110..c81c66ca798 100644 --- a/lib/api/ml/mlflow/api_helpers.rb +++ b/lib/api/ml/mlflow/api_helpers.rb @@ -5,6 +5,7 @@ module API module Mlflow module ApiHelpers OUTER_QUOTES_REGEXP = /^("|')|("|')?$/ + GITLAB_TAG_PREFIX = 'gitlab.' def check_api_read! not_found! unless can?(current_user, :read_model_experiments, user_project) @@ -113,6 +114,27 @@ module API { name: filter } end + def gitlab_tags + return unless params[:tags].present? + + tags = params[:tags] + gitlab_params = {} + + tags.each do |tag| + key, value = tag.values_at(:key, :value) + + gitlab_params[key.delete_prefix(GITLAB_TAG_PREFIX)] = value if key&.starts_with?(GITLAB_TAG_PREFIX) + end + + gitlab_params + end + + def custom_version + return unless gitlab_tags + + gitlab_tags['version'] + end + def find_experiment!(iid, name) experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found! end diff --git a/lib/api/ml/mlflow/experiments.rb b/lib/api/ml/mlflow/experiments.rb index 1a501291941..511922782e8 100644 --- a/lib/api/ml/mlflow/experiments.rb +++ b/lib/api/ml/mlflow/experiments.rb @@ -47,6 +47,42 @@ module API present response, with: Entities::Ml::Mlflow::ListExperiment end + desc 'Search experiments' do + success Entities::Ml::Mlflow::ListExperiment + detail 'https://www.mlflow.org/docs/latest/rest-api.html#list-experiments' + end + params do + optional :max_results, + type: Integer, + desc: 'Maximum number of experiments to fetch in a page. Default is 200, maximum is 1000.', + default: 200 + optional :order_by, + type: String, + desc: 'Order criteria. Can be by a column of the experiment (created_at, name).', + default: 'created_at DESC' + optional :page_token, + type: String, + desc: 'Token for pagination' + optional :filter, + type: String, + desc: 'This parameter is ignored' + end + post 'search', urgency: :low do + max_results = [params[:max_results], 1000].min + + finder_params = model_order_params(params) + + finder = ::Projects::Ml::ExperimentFinder.new(user_project, finder_params) + paginator = finder.execute.keyset_paginate(cursor: params[:page_token], per_page: max_results) + + result = { + experiments: paginator.records, + next_page_token: paginator.cursor_for_next_page + } + + present result, with: Entities::Ml::Mlflow::SearchExperiments + end + desc 'Create experiment' do success Entities::Ml::Mlflow::NewExperiment detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment' diff --git a/lib/api/ml/mlflow/model_versions.rb b/lib/api/ml/mlflow/model_versions.rb index 4b211cf540c..53ba4ff36f1 100644 --- a/lib/api/ml/mlflow/model_versions.rb +++ b/lib/api/ml/mlflow/model_versions.rb @@ -27,13 +27,16 @@ module API desc: 'Register model under this name This field is required.' optional :description, type: String, desc: 'Optional description for model version.' + optional :tags, type: Array, desc: 'Additional metadata for a model version.' end post 'create', urgency: :low do present ::Ml::CreateModelVersionService.new( model, { model_name: params[:name], - description: params[:description] + description: params[:description], + metadata: params[:tags], + version: custom_version } ).execute, with: Entities::Ml::Mlflow::ModelVersion, diff --git a/lib/api/ml/mlflow/registered_models.rb b/lib/api/ml/mlflow/registered_models.rb index a68a2767a74..3f4996a94c0 100644 --- a/lib/api/ml/mlflow/registered_models.rb +++ b/lib/api/ml/mlflow/registered_models.rb @@ -31,13 +31,17 @@ module API optional :tags, type: Array, desc: 'Additional metadata for registered model.' end post 'create', urgency: :low do - present ::Ml::CreateModelService.new( + model = ::Ml::CreateModelService.new( user_project, params[:name], current_user, params[:description], params[:tags] - ).execute, + ).execute + + resource_already_exists! unless model.persisted? + + present model, with: Entities::Ml::Mlflow::RegisteredModel, root: :registered_model rescue ActiveRecord::RecordInvalid diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 750dc7fc2a1..17425c288fc 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -34,6 +34,7 @@ module API params do optional :search, type: String, desc: 'Returns a list of namespaces the user is authorized to view based on the search criteria' optional :owned_only, type: Boolean, desc: 'In GitLab 14.2 and later, returns a list of owned namespaces only' + optional :top_level_only, type: Boolean, default: false, desc: 'Only include top level namespaces' use :pagination use :optional_list_params_ee @@ -43,6 +44,8 @@ module API namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) + namespaces = namespaces.top_most if params[:top_level_only] + namespaces = namespaces.without_project_namespaces.include_route namespaces = namespaces.include_gitlab_subscription_with_hosted_plan if Gitlab.ee? diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index e1d0455b1e2..c6c99944ca0 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true module API class NpmProjectPackages < ::API::Base + ERROR_REASON_TO_HTTP_STATUS_MAPPTING = { + ::Packages::Npm::CreatePackageService::ERROR_REASON_INVALID_PARAMETER => 400, + ::Packages::Npm::CreatePackageService::ERROR_REASON_PACKAGE_LEASE_TAKEN => 400, + ::Packages::Npm::CreatePackageService::ERROR_REASON_PACKAGE_EXISTS => 403, + ::Packages::Npm::CreatePackageService::ERROR_REASON_PACKAGE_PROTECTED => 403 + }.freeze + helpers ::API::Helpers::Packages::Npm feature_category :package_registry @@ -14,6 +21,10 @@ module API def endpoint_scope :project end + + def error_reason_to_http_status(reason) + ERROR_REASON_TO_HTTP_STATUS_MAPPTING.fetch(reason, 400) + end end params do @@ -74,12 +85,13 @@ module API else authorize_create_package!(project) - created_package = ::Packages::Npm::CreatePackageService + service_response = ::Packages::Npm::CreatePackageService .new(project, current_user, params.merge(build: current_authenticated_job)).execute - if created_package[:status] == :error - render_structured_api_error!({ message: created_package[:message], error: created_package[:message] }, created_package[:http_status]) + if service_response.error? + render_structured_api_error!({ message: service_response.message, error: service_response.message }, error_reason_to_http_status(service_response.reason)) else + created_package = service_response[:package] enqueue_sync_metadata_cache_worker(project, created_package.name) track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, namespace: project.namespace) created_package diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 0cedf7d975f..30e126b34cb 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -6,7 +6,6 @@ module API before do require_pages_config_enabled! - authenticated_with_can_read_all_resources! end params do @@ -24,12 +23,30 @@ module API tags %w[pages] end delete ':id/pages' do + authenticated_with_can_read_all_resources! authorize! :remove_pages, user_project ::Pages::DeleteService.new(user_project, current_user).execute no_content! end + + desc 'Get pages settings' do + detail 'Get pages URL and other settings. This feature was introduced in Gitlab 16.8' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 404, message: 'Not Found' } + ] + tags %w[pages] + end + get ':id/pages' do + authorize! :read_pages, user_project + + break not_found! unless user_project.pages_enabled? + + present ::Pages::ProjectSettings.new(user_project), with: Entities::Pages::ProjectSettings + end end end end diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/namespace_packages.rb index 9e82a849c98..1999fc42aba 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/namespace_packages.rb @@ -4,7 +4,7 @@ module API module Terraform module Modules module V1 - class Packages < ::API::Base + class NamespacePackages < ::API::Base include ::API::Helpers::Authentication helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers @@ -29,8 +29,10 @@ module API end helpers do + include ::Gitlab::Utils::StrongMemoize + params :module_name do - requires :module_name, type: String, desc: "", regexp: API::NO_SLASH_URL_PART_REGEX + requires :module_name, type: String, desc: '', regexp: API::NO_SLASH_URL_PART_REGEX requires :module_system, type: String, regexp: API::NO_SLASH_URL_PART_REGEX end @@ -39,10 +41,9 @@ module API end def module_namespace - strong_memoize(:module_namespace) do - find_namespace(params[:module_namespace]) - end + find_namespace(params[:module_namespace]) end + strong_memoize_attr :module_namespace def finder_params { @@ -55,26 +56,23 @@ module API end def packages - strong_memoize(:packages) do - ::Packages::GroupPackagesFinder.new( - current_user, - module_namespace, - finder_params - ).execute - end + ::Packages::GroupPackagesFinder.new( + current_user, + module_namespace, + finder_params + ).execute end + strong_memoize_attr :packages def package - strong_memoize(:package) do - packages.first - end + packages.first end + strong_memoize_attr :package def package_file - strong_memoize(:package_file) do - package.installable_package_files.first - end + package.installable_package_files.first end + strong_memoize_attr :package_file end params do @@ -82,7 +80,8 @@ module API includes :module_name end - namespace 'packages/terraform/modules/v1/:module_namespace/:module_name/:module_system', requirements: TERRAFORM_MODULE_REQUIREMENTS do + namespace 'packages/terraform/modules/v1/:module_namespace/:module_name/:module_system', + requirements: TERRAFORM_MODULE_REQUIREMENTS do authenticate_with do |accept| accept.token_types(:personal_access_token, :deploy_token, :job_token) .sent_through(:http_bearer_token) @@ -118,7 +117,9 @@ module API get 'download' do latest_version = packages.order_version.last&.version - render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) if latest_version.nil? + if latest_version.nil? + render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) + end download_path = api_v4_packages_terraform_modules_v1_module_version_download_path( { @@ -145,7 +146,9 @@ module API get do latest_package = packages.order_version.last - render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) if latest_package&.version.nil? + if latest_package&.version.nil? + render_api_error!({ error: "No version found for #{params[:module_name]} module" }, :not_found) + end presenter = ::Terraform::ModuleVersionPresenter.new(latest_package, params[:module_system]) present presenter, with: ::API::Entities::Terraform::ModuleVersion @@ -181,13 +184,18 @@ module API jwt_token = Gitlab::TerraformRegistryToken.from_token(token_from_namespace_inheritable).encoded end - header 'X-Terraform-Get', module_file_path.sub(%r{module_version/file$}, "#{params[:module_version]}/file?token=#{jwt_token}&archive=tgz") + header 'X-Terraform-Get', + module_file_path.sub( + %r{module_version/file$}, + "#{params[:module_version]}/file?token=#{jwt_token}&archive=tgz" + ) status :no_content end namespace 'file' do authenticate_with do |accept| - accept.token_types(:deploy_token_from_jwt, :job_token_from_jwt, :personal_access_token_from_jwt).sent_through(:token_param) + accept.token_types(:deploy_token_from_jwt, :job_token_from_jwt, :personal_access_token_from_jwt) + .sent_through(:token_param) end desc 'Download specific version of a module' do @@ -200,9 +208,14 @@ module API tags %w[terraform_registry] end get do - track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace) - - present_carrierwave_file!(package_file.file) + track_package_event( + 'pull_package', + :terraform_module, + project: package.project, + namespace: module_namespace + ) + + present_package_file!(package_file) end end diff --git a/lib/api/terraform/modules/v1/project_packages.rb b/lib/api/terraform/modules/v1/project_packages.rb index 07dfddefefc..c0a84c7b36c 100644 --- a/lib/api/terraform/modules/v1/project_packages.rb +++ b/lib/api/terraform/modules/v1/project_packages.rb @@ -16,87 +16,174 @@ module API require_packages_enabled! end - params do - requires :id, type: String, desc: 'The ID or full path of a project' - requires :module_name, type: String, desc: "", regexp: API::NO_SLASH_URL_PART_REGEX - requires :module_system, type: String, regexp: API::NO_SLASH_URL_PART_REGEX - requires :module_version, type: String, desc: 'Module version', regexp: Gitlab::Regex.semver_regex - end + helpers do + params :terraform_get do + optional 'terraform-get', type: String, values: %w[1], desc: 'Terraform get redirection flag' + end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - namespace ':id/packages/terraform/modules/:module_name/:module_system/*module_version/file' do - authenticate_with do |accept| - accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) - accept.token_types(:job_token).sent_through(:http_job_token_header) - accept.token_types(:personal_access_token).sent_through(:http_private_token_header) + def present_package_file + authorize_read_package!(authorized_user_project) + + if declared_params[:'terraform-get'] == '1' + header 'X-Terraform-Get', "#{request.url.split('?').first}?archive=tgz" + return no_content! end - desc 'Workhorse authorize Terraform Module package file' do - detail 'This feature was introduced in GitLab 13.11' - success code: 200 - failure [ - { code: 403, message: 'Forbidden' } - ] - tags %w[terraform_registry] + package = ::Packages::TerraformModule::PackagesFinder + .new(authorized_user_project, finder_params) + .execute + .first + + not_found! unless package + + track_package_event('pull_package', :terraform_module, project: authorized_user_project, + namespace: authorized_user_project.namespace) + + present_package_file!(package.installable_package_files.first) + end + + def finder_params + { package_name: package_name }.tap do |finder_params| + finder_params[:package_version] = params[:module_version] if params.key?(:module_version) end + end + + def package_name + "#{params[:module_name]}/#{params[:module_system]}" + end + end - put 'authorize' do - authorize_workhorse!( - subject: authorized_user_project, - maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size - ) + params do + requires :id, types: [String, Integer], allow_blank: false, desc: 'The ID or full path of a project' + with(type: String, allow_blank: false, regexp: API::NO_SLASH_URL_PART_REGEX) do + requires :module_name, desc: 'Module name', documentation: { example: 'infra-registry' } + requires :module_system, desc: 'Module system', documentation: { example: 'aws' } + end + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/terraform/modules/:module_name/:module_system' do + authenticate_with do |accept| + accept.token_types( + :personal_access_token_with_username, + :deploy_token_with_username, + :job_token_with_username + ).sent_through(:http_basic_auth) end - desc 'Upload Terraform Module package file' do - detail 'This feature was introduced in GitLab 13.11' - success code: 201 + desc 'Download the latest version of a module' do + detail 'This feature was introduced in GitLab 16.7' + success code: 204 failure [ - { code: 400, message: 'Invalid file' }, { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, { code: 404, message: 'Not found' } ] - consumes %w[multipart/form-data] tags %w[terraform_registry] end - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, - desc: 'The package file to be published (generated by Multipart middleware)', - documentation: { type: 'file' } + use :terraform_get + end + get do + present_package_file end - put do - authorize_upload!(authorized_user_project) - - bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?( - :terraform_module_max_file_size, params[:file].size) - - create_package_file_params = { - module_name: params['module_name'], - module_system: params['module_system'], - module_version: params['module_version'], - file: params['file'], - build: current_authenticated_job - } - - result = ::Packages::TerraformModule::CreatePackageService - .new(authorized_user_project, current_user, create_package_file_params) - .execute - - render_api_error!(result[:message], result[:http_status]) if result[:status] == :error - - track_package_event('push_package', :terraform_module, project: authorized_user_project, - namespace: authorized_user_project.namespace) - - created! - rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception( - e, - extra: { file_name: params[:file_name], project_id: authorized_user_project.id } - ) - - forbidden! + params do + requires :module_version, type: String, allow_blank: false, desc: 'Module version', + regexp: Gitlab::Regex.semver_regex + end + namespace '*module_version' do + desc 'Download a specific version of a module' do + detail 'This feature was introduced in GitLab 16.7' + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + tags %w[terraform_registry] + end + params do + use :terraform_get + end + get format: false do + present_package_file + end + + namespace :file do + authenticate_with do |accept| + accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) + accept.token_types(:job_token).sent_through(:http_job_token_header) + accept.token_types(:personal_access_token).sent_through(:http_private_token_header) + end + + desc 'Workhorse authorize Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' } + ] + tags %w[terraform_registry] + end + + put :authorize do + authorize_workhorse!( + subject: authorized_user_project, + maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size + ) + end + + desc 'Upload Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + success code: 201 + failure [ + { code: 400, message: 'Invalid file' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + consumes %w[multipart/form-data] + tags %w[terraform_registry] + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, + desc: 'The package file to be published (generated by Multipart middleware)', + documentation: { type: 'file' } + end + + put do + authorize_upload!(authorized_user_project) + + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?( + :terraform_module_max_file_size, params[:file].size + ) + + create_package_file_params = { + module_name: params['module_name'], + module_system: params['module_system'], + module_version: params['module_version'], + file: params['file'], + build: current_authenticated_job + } + + result = ::Packages::TerraformModule::CreatePackageService + .new(authorized_user_project, current_user, create_package_file_params) + .execute + + render_api_error!(result.message, result.reason) if result.error? + + track_package_event('push_package', :terraform_module, project: authorized_user_project, + namespace: authorized_user_project.namespace) + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, + extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) + + forbidden! + end + end end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 38fa247055e..8b54fb84dd2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1077,8 +1077,6 @@ module API end end - helpers Helpers::UserPreferencesHelpers - desc "Get the currently authenticated user's SSH keys" do success Entities::SSHKey end @@ -1269,9 +1267,7 @@ module API optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' optional :pass_user_identities_to_ci_jwt, type: Boolean, desc: 'Flag indicating the user passes their external identities to a CI job as part of a JSON web token.' - optional :code_suggestions, type: Boolean, desc: 'Flag indicating the user allows code suggestions.' \ - 'Argument is experimental and can be removed in the future without notice.' - at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt, :code_suggestions + at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt end put "preferences", feature_category: :user_profile, urgency: :high do authenticate! @@ -1280,8 +1276,6 @@ module API attrs = declared_params(include_missing: false) - attrs = update_user_namespace_settings(attrs) - render_api_error!('400 Bad Request', 400) unless attrs service = ::UserPreferences::UpdateService.new(current_user, attrs).execute diff --git a/lib/backup/database_configuration.rb b/lib/backup/database_configuration.rb index 1a6a476f9c1..a9e3d175c30 100644 --- a/lib/backup/database_configuration.rb +++ b/lib/backup/database_configuration.rb @@ -19,7 +19,8 @@ module Backup @connection_name = connection_name @source_model = Gitlab::Database.database_base_models_with_gitlab_shared[connection_name] || Gitlab::Database.database_base_models_with_gitlab_shared['main'] - @activerecord_database_config = ActiveRecord::Base.configurations.find_db_config(connection_name) + @activerecord_database_config = ActiveRecord::Base.configurations.find_db_config(connection_name) || + ActiveRecord::Base.configurations.find_db_config('main') end # ENV variables that can override each database configuration diff --git a/lib/backup/database_model.rb b/lib/backup/database_model.rb deleted file mode 100644 index 228a7fa5383..00000000000 --- a/lib/backup/database_model.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -module Backup - class DatabaseModel - SUPPORTED_OVERRIDES = { - username: 'PGUSER', - host: 'PGHOST', - port: 'PGPORT', - password: 'PGPASSWORD', - # SSL - sslmode: 'PGSSLMODE', - sslkey: 'PGSSLKEY', - sslcert: 'PGSSLCERT', - sslrootcert: 'PGSSLROOTCERT', - sslcrl: 'PGSSLCRL', - sslcompression: 'PGSSLCOMPRESSION' - }.freeze - - OVERRIDE_PREFIXES = %w[GITLAB_BACKUP_ GITLAB_OVERRIDE_].freeze - - attr_reader :config - - def initialize(name) - configure_model(name) - end - - def connection - @model.connection - end - - private - - def configure_model(name) - source_model = Gitlab::Database.database_base_models_with_gitlab_shared[name] || - Gitlab::Database.database_base_models_with_gitlab_shared['main'] - - @model = backup_model_for(name) - - original_config = source_model.connection_db_config.configuration_hash.dup - - @config = config_for_backup(name, original_config) - - @model.establish_connection( - ActiveRecord::DatabaseConfigurations::HashConfig.new( - source_model.connection_db_config.env_name, - name.to_s, - original_config.merge(@config[:activerecord]) - ) - ) - - Gitlab::Database::LoadBalancing::Setup.new(@model).setup - end - - def backup_model_for(name) - klass_name = name.camelize - - return "#{self.class.name}::#{klass_name}".constantize if self.class.const_defined?(klass_name.to_sym, false) - - self.class.const_set(klass_name, Class.new(ApplicationRecord)) - end - - def config_for_backup(name, config) - db_config = { - activerecord: config, - pg_env: {} - } - SUPPORTED_OVERRIDES.each do |opt, arg| - # This enables the use of different PostgreSQL settings in - # case PgBouncer is used. PgBouncer clears the search path, - # which wreaks havoc on Rails if connections are reused. - OVERRIDE_PREFIXES.each do |override_prefix| - override_all = "#{override_prefix}#{arg}" - override_db = "#{override_prefix}#{name.upcase}_#{arg}" - val = ENV[override_db].presence || ENV[override_all].presence || config[opt].to_s.presence - - next unless val - - db_config[:pg_env][arg] = val - db_config[:activerecord][opt] = val - end - end - - db_config - end - end -end diff --git a/lib/banzai/filter/markdown_engines/base.rb b/lib/banzai/filter/markdown_engines/base.rb index 34f1d4d3da9..66d6440e257 100644 --- a/lib/banzai/filter/markdown_engines/base.rb +++ b/lib/banzai/filter/markdown_engines/base.rb @@ -4,8 +4,10 @@ module Banzai module Filter module MarkdownEngines class Base + attr_reader :context + def initialize(context) - @context = context + @context = context || {} end def render(text) @@ -15,7 +17,7 @@ module Banzai private def sourcepos_disabled? - @context[:no_sourcepos] + context[:no_sourcepos] end end end diff --git a/lib/banzai/filter/markdown_engines/glfm_markdown.rb b/lib/banzai/filter/markdown_engines/glfm_markdown.rb new file mode 100644 index 00000000000..40539ac7961 --- /dev/null +++ b/lib/banzai/filter/markdown_engines/glfm_markdown.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'glfm_markdown' + +# Use the glfm_markdown gem (https://gitlab.com/gitlab-org/ruby/gems/gitlab-glfm-markdown) +# to interface with the Rust based `comrak` parser +# https://github.com/kivikakk/comrak +module Banzai + module Filter + module MarkdownEngines + class GlfmMarkdown < Base + OPTIONS = { + autolink: true, + footnotes: true, + full_info_string: true, + github_pre_lang: true, + hardbreaks: false, + relaxed_autolinks: false, + sourcepos: true, + smart: false, + strikethrough: true, + table: true, + tagfilter: false, + tasklist: false, # still handled by a banzai filter/gem + unsafe: true + }.freeze + + def render(text) + ::GLFMMarkdown.to_html(text, options: render_options) + end + + private + + def render_options + sourcepos_disabled? ? OPTIONS.merge(sourcepos: false) : OPTIONS + end + end + end + end +end + +Banzai::Filter::MarkdownEngines::GlfmMarkdown.prepend_mod diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index e6a0cdfe020..5f442ed12d4 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -3,12 +3,13 @@ module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter - DEFAULT_ENGINE = :common_mark + RUST_ENGINE = :glfm_markdown # glfm_markdown/comrak + RUBY_ENGINE = :common_mark # original commonmarker/cmark-gfm def initialize(text, context = nil, result = nil) super(text, context, result) - @renderer = self.class.render_engine(context[:markdown_engine]).new(context) + @renderer = render_engine.new(@context) @text = @text.delete("\r") end @@ -16,13 +17,27 @@ module Banzai @renderer.render(@text).rstrip end - class << self - def render_engine(engine_from_context) - "Banzai::Filter::MarkdownEngines::#{engine(engine_from_context)}".constantize - rescue NameError - raise NameError, "`#{engine_from_context}` is unknown markdown engine" - end + def render_engine + "Banzai::Filter::MarkdownEngines::#{engine}".constantize + rescue NameError + raise NameError, "`#{engine_class}` is unknown markdown engine" + end + + private + + def engine + engine = context[:markdown_engine] || default_engine + + engine.to_s.classify + end + + def default_engine + return RUST_ENGINE if Feature.enabled?(:markdown_rust, context[:project]) + RUBY_ENGINE + end + + class << self # Parses string representing a sourcepos in format # "start_row:start_column-end_row:end_column" into 0-based # attributes. For example, "1:10-14:1" becomes @@ -42,14 +57,6 @@ module Banzai end: { row: [1, end_row.to_i].max - 1, col: [1, end_col.to_i].max - 1 } } end - - private - - def engine(engine_from_context) - engine_from_context ||= DEFAULT_ENGINE - - engine_from_context.to_s.classify - end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 1b3905f0dde..474efe71b70 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -53,11 +53,11 @@ module Banzai Filter::References::MergeRequestReferenceFilter, Filter::References::SnippetReferenceFilter, Filter::References::CommitRangeReferenceFilter, - Filter::References::CommitReferenceFilter, Filter::References::LabelReferenceFilter, Filter::References::MilestoneReferenceFilter, Filter::References::AlertReferenceFilter, - Filter::References::FeatureFlagReferenceFilter + Filter::References::FeatureFlagReferenceFilter, + Filter::References::CommitReferenceFilter ] end diff --git a/lib/bulk_imports/file_downloads/validations.rb b/lib/bulk_imports/file_downloads/validations.rb index 14f036e469c..261603da2ea 100644 --- a/lib/bulk_imports/file_downloads/validations.rb +++ b/lib/bulk_imports/file_downloads/validations.rb @@ -38,20 +38,14 @@ module BulkImports raise_error 'Invalid downloaded file' end - def validate_content_length - validate_size!(response_headers['content-length']) - end - def validate_size!(size) - if size.blank? - raise_error 'Missing content-length header' - elsif file_size_limit > 0 && size.to_i > file_size_limit - raise_error format( - "File size %{size} exceeds limit of %{limit}", - size: ActiveSupport::NumberHelper.number_to_human_size(size), - limit: ActiveSupport::NumberHelper.number_to_human_size(file_size_limit) - ) - end + return unless file_size_limit > 0 && size.to_i > file_size_limit + + raise_error format( + "File size %{size} exceeds limit of %{limit}", + size: ActiveSupport::NumberHelper.number_to_human_size(size), + limit: ActiveSupport::NumberHelper.number_to_human_size(file_size_limit) + ) end end end diff --git a/lib/click_house/iterator.rb b/lib/click_house/iterator.rb index 4bfbc624dc7..f17f3efa8a5 100644 --- a/lib/click_house/iterator.rb +++ b/lib/click_house/iterator.rb @@ -22,9 +22,10 @@ module ClickHouse # builder = ClickHouse::QueryBuilder.new('event_authors').where(type: 'some_type') class Iterator # rubocop: disable CodeReuse/ActiveRecord -- this is a ClickHouse query builder class usin Arel - def initialize(query_builder:, connection:) + def initialize(query_builder:, connection:, min_value: nil) @query_builder = query_builder @connection = connection + @min_value = min_value end def each_batch(column: :id, of: 10_000) @@ -36,18 +37,18 @@ module ClickHouse row = connection.select(min_max_query.to_sql).first return if row.nil? - min_value = row['min'] - max_value = row['max'] - return if max_value == 0 + min = min_value || row['min'] + max = row['max'] + return if max == 0 loop do - break if min_value > max_value + break if min > max yield query_builder - .where(table[column].gteq(min_value)) - .where(table[column].lt(min_value + of)) + .where(table[column].gteq(min)) + .where(table[column].lt(min + of)) - min_value += of + min += of end end @@ -55,7 +56,7 @@ module ClickHouse delegate :table, to: :query_builder - attr_reader :query_builder, :connection + attr_reader :query_builder, :connection, :min_value # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 9b6c37da847..276f9b492cb 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -170,7 +170,7 @@ module ContainerRegistry end # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/gitlab/api.md#list-repository-tags - def tags(path, page_size: 100, last: nil, before: nil, name: nil, sort: nil) + def tags(path, page_size: 100, last: nil, before: nil, name: nil, sort: nil, referrers: nil) limited_page_size = [page_size, MAX_TAGS_PAGE_SIZE].min with_token_faraday do |faraday_client| url = "#{GITLAB_REPOSITORIES_PATH}/#{path}/tags/list/" @@ -180,6 +180,7 @@ module ContainerRegistry req.params['before'] = before if before req.params['name'] = name if name.present? req.params['sort'] = sort if sort + req.params['referrers'] = 'true' if referrers end unless response.success? diff --git a/lib/container_registry/referrer.rb b/lib/container_registry/referrer.rb new file mode 100644 index 00000000000..e61899b3a1b --- /dev/null +++ b/lib/container_registry/referrer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module ContainerRegistry + class Referrer + attr_reader :artifact_type, :digest, :tag + + def initialize(artifact_type, digest, tag) + @artifact_type = artifact_type + @digest = digest + @tag = tag + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 70742e8bd38..4cf4cd71f86 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -4,7 +4,7 @@ module ContainerRegistry class Tag include Gitlab::Utils::StrongMemoize - attr_reader :repository, :name, :updated_at + attr_reader :repository, :name, :updated_at, :referrers, :published_at attr_writer :created_at, :manifest_digest, :revision, :total_size delegate :registry, :client, to: :repository @@ -14,6 +14,10 @@ module ContainerRegistry @name = name end + def referrers=(refs) + @referrers = Array.wrap(refs).map { |ref| Referrer.new(ref['artifactType'], ref['digest'], self) } + end + def revision @revision || config_blob&.revision end @@ -97,24 +101,20 @@ module ContainerRegistry # this function will set and memoize a created_at # to avoid a #config_blob call. def force_created_at_from_iso8601(string_value) - date = - begin - DateTime.iso8601(string_value) - rescue ArgumentError - nil - end + date = parse_iso8601_string(string_value) instance_variable_set(ivar(:memoized_created_at), date) end def updated_at=(string_value) return unless string_value - @updated_at = - begin - DateTime.iso8601(string_value) - rescue ArgumentError - nil - end + @updated_at = parse_iso8601_string(string_value) + end + + def published_at=(string_value) + return unless string_value + + @published_at = parse_iso8601_string(string_value) end def layers @@ -151,5 +151,13 @@ module ContainerRegistry client.delete_repository_tag_by_digest(repository.path, digest) end + + private + + def parse_iso8601_string(string_value) + DateTime.iso8601(string_value) + rescue ArgumentError + nil + end end end diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index af60fb95c53..14848f22f83 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -49,23 +49,27 @@ module Feature end unless type.present? - raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is missing type. Ensure to update #{path}" + raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is missing `type`. Ensure to update #{path}" end unless Definition::TYPES.include?(type.to_sym) raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' type '#{type}' is invalid. Ensure to update #{path}" end - unless File.basename(path, ".yml") == name + if File.basename(path, ".yml") != name || File.basename(File.dirname(path)) != type raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' has an invalid path: '#{path}'. Ensure to update #{path}" end - unless File.basename(File.dirname(path)) == type - raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' has an invalid type: '#{path}'. Ensure to update #{path}" - end + validate_default_enabled! + end + def validate_default_enabled! if default_enabled.nil? - raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is missing default_enabled. Ensure to update #{path}" + raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' is missing `default_enabled`. Ensure to update #{path}" + end + + if default_enabled && !Definition::TYPES.dig(type.to_sym, :can_be_default_enabled) + raise Feature::InvalidFeatureFlagError, "Feature flag '#{name}' cannot have `default_enabled` set to `true`. Ensure to update #{path}" end end diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index d801070ff1a..fcce2642ef6 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -8,21 +8,39 @@ module Feature module Shared # optional: defines if a on-disk definition is required for this feature flag type # rollout_issue: defines if `bin/feature-flag` asks for rollout issue - # default_enabled: defines a default state of a feature flag when created by `bin/feature-flag` - # ee_only: defines that a feature flag can only be created in a context of EE + # can_be_default_enabled: whether the flag can have `default_enabled` set to `true` or not # deprecated: defines if a feature flag type that is deprecated and to be removed, # the deprecated types are hidden from all interfaces # example: usage being shown when exception is raised TYPES = { - development: { - description: 'Short lived, used to enable unfinished code to be deployed', + gitlab_com_derisk: { + description: 'Short lived, used to de-risk GitLab.com deployments', + optional: false, + rollout_issue: true, + can_be_default_enabled: false, + example: <<-EOS + Feature.enabled?(:my_feature_flag, project, type: :gitlab_com_derisk) + push_frontend_feature_flag(:my_feature_flag, project) + EOS + }, + wip: { + description: 'Used to hide unfinished code from anyone', + optional: false, + rollout_issue: false, + can_be_default_enabled: false, + example: <<-EOS + Feature.enabled?(:my_feature_flag, project, type: :wip) + push_frontend_feature_flag(:my_feature_flag, project) + EOS + }, + beta: { + description: "Use when we aren't confident about scaling/supporting a feature, " \ + "or when it isn't complete enough for an MVC", optional: false, rollout_issue: true, - ee_only: false, - default_enabled: false, + can_be_default_enabled: true, example: <<-EOS - Feature.enabled?(:my_feature_flag, project) - Feature.enabled?(:my_feature_flag, project, type: :development) + Feature.enabled?(:my_feature_flag, project, type: :beta) push_frontend_feature_flag(:my_feature_flag, project) EOS }, @@ -30,27 +48,17 @@ module Feature description: "Long-lived feature flags that control operational aspects of GitLab's behavior", optional: false, rollout_issue: true, - ee_only: false, - default_enabled: false, + can_be_default_enabled: true, example: <<-EOS Feature.enabled?(:my_ops_flag, type: :ops) push_frontend_feature_flag(:my_ops_flag, project, type: :ops) EOS }, - undefined: { - description: "Feature flags that are undefined in GitLab codebase (should not be used)", - optional: true, - rollout_issue: false, - ee_only: false, - default_enabled: false, - example: '' - }, experiment: { description: 'Short lived, used specifically to run A/B/n experiments.', optional: true, rollout_issue: true, - ee_only: false, - default_enabled: false, + can_be_default_enabled: false, example: <<-EOS experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } EOS @@ -59,12 +67,22 @@ module Feature description: "Feature flags for controlling Sidekiq workers behavior (e.g. deferring jobs)", optional: true, rollout_issue: false, - ee_only: false, - default_enabled: false, + can_be_default_enabled: false, example: '<<-EOS Feature.enabled?(:"defer_sidekiq_jobs:AuthorizedProjectsWorker", type: :worker, default_enabled_if_undefined: false) EOS' + }, + undefined: { + description: "Feature flags that are undefined in GitLab codebase (should not be used)", + optional: true, + rollout_issue: false, + can_be_default_enabled: false, + example: '' + }, + development: { + deprecated: true, + can_be_default_enabled: true } }.freeze @@ -72,6 +90,7 @@ module Feature # This is done to ease the file comparison PARAMS = %i[ name + feature_issue_url introduced_by_url rollout_issue_url milestone diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 3d2f13af9dc..5a2881e6c96 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -30,7 +30,7 @@ module Gitlab group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, - member_delete: { threshold: 60, interval: 1.minute }, + members_delete: { threshold: -> { application_settings.members_delete_limit }, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, web_hook_calls: { interval: 1.minute }, web_hook_calls_mid: { interval: 1.minute }, @@ -52,7 +52,7 @@ module Gitlab project_testing_integration: { threshold: 5, interval: 1.minute }, email_verification: { threshold: 10, interval: 10.minutes }, email_verification_code_send: { threshold: 10, interval: 1.hour }, - phone_verification_challenge: { threshold: 3, interval: 1.day }, + phone_verification_challenge: { threshold: 2, interval: 1.day }, phone_verification_send_code: { threshold: 5, interval: 1.day }, phone_verification_verify_code: { threshold: 5, interval: 1.day }, namespace_exists: { threshold: 20, interval: 1.minute }, diff --git a/lib/gitlab/application_setting_fetcher.rb b/lib/gitlab/application_setting_fetcher.rb new file mode 100644 index 00000000000..cc8f67dc541 --- /dev/null +++ b/lib/gitlab/application_setting_fetcher.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationSettingFetcher + class << self + def clear_in_memory_application_settings! + @in_memory_application_settings = nil + end + + def current_application_settings + cached_application_settings || uncached_application_settings + end + + def current_application_settings? + ::ApplicationSetting.current.present? + end + + def expire_current_application_settings + ::ApplicationSetting.expire + end + + private + + def cached_application_settings + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + + begin + ::ApplicationSetting.cached + rescue StandardError + # In case Redis isn't running + # or the Redis UNIX socket file is not available + # or the DB is not running (we use migrations in the cache key) + end + end + + def uncached_application_settings + return fake_application_settings if Gitlab::Runtime.rake? && !connect_to_db? + + current_settings = ::ApplicationSetting.current + + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if Gitlab::Runtime.rake? && ::ApplicationSetting.connection.migration_context.needs_migration? + db_attributes = current_settings&.attributes || {} + fake_application_settings(db_attributes) + elsif current_settings.present? + current_settings + else + ::ApplicationSetting.create_from_defaults + end + rescue ::ApplicationSetting::Recursion + in_memory_application_settings + end + + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults + end + + def fake_application_settings(attributes = {}) + Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) + end + + def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = begin + ::ApplicationSetting.connection.active? + rescue StandardError + false + end + + active_db_connection && + ApplicationSetting.database.cached_table_exists? + rescue ActiveRecord::NoDatabaseError + false + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8e894be4fc4..bdd1aed4017 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -43,10 +43,13 @@ module Gitlab WRITE_OBSERVABILITY_SCOPE = :write_observability OBSERVABILITY_SCOPES = [READ_OBSERVABILITY_SCOPE, WRITE_OBSERVABILITY_SCOPE].freeze + # Scopes for Monitor access + READ_SERVICE_PING_SCOPE = :read_service_ping + # Scopes used for GitLab as admin SUDO_SCOPE = :sudo ADMIN_MODE_SCOPE = :admin_mode - ADMIN_SCOPES = [SUDO_SCOPE, ADMIN_MODE_SCOPE].freeze + ADMIN_SCOPES = [SUDO_SCOPE, ADMIN_MODE_SCOPE, READ_SERVICE_PING_SCOPE].freeze # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [API_SCOPE].freeze diff --git a/lib/gitlab/auth/two_factor_auth_verifier.rb b/lib/gitlab/auth/two_factor_auth_verifier.rb index 4b66aaf0e6a..971f8bb25ce 100644 --- a/lib/gitlab/auth/two_factor_auth_verifier.rb +++ b/lib/gitlab/auth/two_factor_auth_verifier.rb @@ -14,13 +14,28 @@ module Gitlab two_factor_authentication_required? && two_factor_grace_period_expired? end + # rubocop:disable Cop/UserAdmin -- Admin mode does not matter in the context of verifying for two factor statuses def two_factor_authentication_required? return false if allow_2fa_bypass_for_provider Gitlab::CurrentSettings.require_two_factor_authentication? || - current_user&.require_two_factor_authentication_from_group? + current_user&.require_two_factor_authentication_from_group? || + (Gitlab::CurrentSettings.require_admin_two_factor_authentication && current_user&.admin?) # rubocop:disable Cop/UserAdmin -- It should be applied to any administrator user regardless of admin mode end + def two_factor_authentication_reason + if Gitlab::CurrentSettings.require_two_factor_authentication? + :global + elsif Gitlab::CurrentSettings.require_admin_two_factor_authentication && current_user&.admin? + :admin_2fa + elsif current_user&.require_two_factor_authentication_from_group? + :group + else + false + end + end + # rubocop:enable Cop/UserAdmin + def current_user_needs_to_setup_two_factor? current_user && !current_user.temp_oauth_email? && !current_user.two_factor_enabled? end diff --git a/lib/gitlab/background_migration/backfill_issue_search_data_namespace_id.rb b/lib/gitlab/background_migration/backfill_issue_search_data_namespace_id.rb new file mode 100644 index 00000000000..56d69a549dc --- /dev/null +++ b/lib/gitlab/background_migration/backfill_issue_search_data_namespace_id.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Updates issue_search_data.namespace_id with the associated issue's namespace_id + class BackfillIssueSearchDataNamespaceId < BatchedMigrationJob # rubocop:disable Search/NamespacedClass -- This is a migration class + feature_category :team_planning + operation_name :backfill_issue_search_data_namespace_id + + # migrations only version of `issue_search_data` table + class IssueSearchData < ::ApplicationRecord # rubocop:disable Search/NamespacedClass -- Inline class for migration + self.table_name = 'issue_search_data' + end + + def perform + each_sub_batch do |sub_batch| + issues_by_project = sub_batch + .where.not(project_id: nil) + .pluck(:project_id, :namespace_id, :id) + .group_by(&:first) + + issues_by_project.each do |project_id, issues| + namespace_id = issues.first[1] + issue_ids = issues.pluck(2) + + IssueSearchData + .where(issue_id: issue_ids, project_id: project_id) + .update_all(namespace_id: namespace_id) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_owasp_top_ten_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_owasp_top_ten_of_vulnerability_reads.rb new file mode 100644 index 00000000000..5d8867a130a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_owasp_top_ten_of_vulnerability_reads.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills owasp_top_10 column for vulnerability_reads table. + class BackfillOwaspTopTenOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_owasp_top_10 + feature_category :vulnerability_management + + OWASP_TOP_10 = { + "A1:2017-Injection" => 1, + "A1:2017" => 1, + "A2:2017-Broken Authentication" => 2, + "A2:2017" => 2, + "A3:2017-Sensitive Data Exposure" => 3, + "A3:2017" => 3, + "A4:2017-XML External Entities (XXE)" => 4, + "A4:2017" => 4, + "A5:2017-Broken Access Control" => 5, + "A5:2017" => 5, + "A6:2017-Security Misconfiguration" => 6, + "A6:2017" => 6, + "A7:2017-Cross-Site Scripting (XSS)" => 7, + "A7:2017" => 7, + "A8:2017-Insecure Deserialization" => 8, + "A8:2017" => 8, + "A9:2017-Using Components with Known Vulnerabilities" => 9, + "A9:2017" => 9, + "A10:2017-Insufficient Logging & Monitoring" => 10, + "A10:2017" => 10, + + "A1:2021-Broken Access Control" => 11, + "A1:2021" => 11, + "A2:2021-Cryptographic Failures" => 12, + "A2:2021" => 12, + "A3:2021-Injection" => 13, + "A3:2021" => 13, + "A4:2021-Insecure Design" => 14, + "A4:2021" => 14, + "A5:2021-Security Misconfiguration" => 15, + "A5:2021" => 15, + "A6:2021-Vulnerable and Outdated Components" => 16, + "A6:2021" => 16, + "A7:2021-Identification and Authentication Failures" => 17, + "A7:2021" => 17, + "A8:2021-Software and Data Integrity Failures" => 18, + "A8:2021" => 18, + "A9:2021-Security Logging and Monitoring Failures" => 19, + "A9:2021" => 19, + "A10:2021-Server-Side Request Forgery" => 20, + "A10:2021" => 20 + }.with_indifferent_access.freeze + + UPDATE_SQL = <<-SQL.squish + UPDATE vulnerability_reads AS vr + SET owasp_top_10 = + CASE selected_ids.external_id + #{OWASP_TOP_10.map { |external_id, value| "WHEN '#{external_id}' THEN #{value}" }.join(' ')} + ELSE vr.owasp_top_10 + END + FROM ( + SELECT vr.id, vi.external_id + FROM vulnerability_reads vr + INNER JOIN vulnerability_occurrences vo ON vr.vulnerability_id = vo.vulnerability_id + INNER JOIN vulnerability_occurrence_identifiers voi ON vo.id = voi.occurrence_id + INNER JOIN vulnerability_identifiers vi ON voi.identifier_id = vi.id + WHERE LOWER(vi.external_type) = 'owasp' + AND vi.external_id IN (?) + AND vr.id IN (?) + ) AS selected_ids + WHERE vr.id = selected_ids.id + SQL + + class VulnerabilitiesRead < ::ApplicationRecord + self.table_name = 'vulnerability_reads' + end + + def perform + each_sub_batch do |sub_batch| + update_query = VulnerabilitiesRead.sanitize_sql([UPDATE_SQL, OWASP_TOP_10.keys, sub_batch.select(:id)]) + connection.execute(update_query) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_artifact.rb b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_artifact.rb new file mode 100644 index 00000000000..73996ce4460 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_artifact.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPartitionIdCiPipelineArtifact < BatchedMigrationJob + operation_name :update_all + feature_category :continuous_integration + + def perform + return unless uses_multiple_partitions? + + each_sub_batch do |sub_batch| + sub_batch + .where('ci_pipeline_artifacts.pipeline_id = ci_pipelines.id') + .update_all('partition_id = ci_pipelines.partition_id FROM ci_pipelines') + end + end + + private + + def uses_multiple_partitions? + !!connection.select_value(<<~SQL) + SELECT true FROM p_ci_builds WHERE partition_id = 101 LIMIT 1 + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_chat_data.rb b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_chat_data.rb new file mode 100644 index 00000000000..e367872245b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_chat_data.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPartitionIdCiPipelineChatData < BatchedMigrationJob + operation_name :update + feature_category :continuous_integration + + def perform + return unless uses_multiple_partitions? + + each_sub_batch do |sub_batch| + sub_batch + .where('ci_pipeline_chat_data.pipeline_id = ci_pipelines.id') + .update_all('partition_id = ci_pipelines.partition_id FROM ci_pipelines') + end + end + + private + + def uses_multiple_partitions? + !!connection.select_value(<<~SQL) + SELECT true FROM p_ci_builds WHERE partition_id = 101 LIMIT 1 + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_config.rb b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_config.rb new file mode 100644 index 00000000000..de20eae8cf0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_config.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPartitionIdCiPipelineConfig < BatchedMigrationJob + operation_name :update_all + feature_category :continuous_integration + scope_to ->(relation) { relation.where('ci_pipelines_config.pipeline_id >= ?', first_pipeline_id) } + + def perform + each_sub_batch do |sub_batch| + sub_batch + .where('ci_pipelines_config.pipeline_id = ci_pipelines.id') + .update_all('partition_id = ci_pipelines.partition_id FROM ci_pipelines') + end + end + + private + + def first_pipeline_id + first_pipeline_with_partition_101 || max_pipeline_id + end + + def first_pipeline_with_partition_101 + connection.select_value(<<~SQL) + SELECT MIN(commit_id) FROM p_ci_builds WHERE partition_id = 101; + SQL + end + + def max_pipeline_id + connection.select_value(<<~SQL) + SELECT MAX(pipeline_id) FROM ci_pipelines_config; + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_metadata.rb b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_metadata.rb new file mode 100644 index 00000000000..6fa4037f1ac --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partition_id_ci_pipeline_metadata.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPartitionIdCiPipelineMetadata < BatchedMigrationJob + operation_name :update_all + feature_category :continuous_integration + + def perform + return unless uses_multiple_partitions? + + each_sub_batch do |sub_batch| + sub_batch + .where('ci_pipeline_metadata.pipeline_id = ci_pipelines.id') + .update_all('partition_id = ci_pipelines.partition_id FROM ci_pipelines') + end + end + + private + + def uses_multiple_partitions? + !!connection.select_value(<<~SQL) + SELECT true FROM p_ci_builds WHERE partition_id = 101 LIMIT 1 + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_vs_code_settings_version.rb b/lib/gitlab/background_migration/backfill_vs_code_settings_version.rb new file mode 100644 index 00000000000..83dbf5f3852 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_vs_code_settings_version.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillVsCodeSettingsVersion < BatchedMigrationJob + feature_category :web_ide + operation_name :backfill_vs_code_settings_version + scope_to ->(relation) { relation.where(version: [nil, 0]) } + + class VsCodeSetting < ApplicationRecord + DEFAULT_SETTING_VERSIONS = { + 'settings' => 2, + 'extensions' => 6, + 'globalState' => 1, + 'keybindings' => 2, + 'snippets' => 1, + 'machines' => 1, + 'tasks' => 1, + 'profiles' => 2 + }.freeze + + self.table_name = 'vs_code_settings' + end + + def perform + each_sub_batch do |sub_batch| + vs_code_settings = sub_batch.map do |vs_code_setting| + version = VsCodeSetting::DEFAULT_SETTING_VERSIONS[vs_code_setting.setting_type] + + vs_code_setting.attributes.merge(version: version) + end + + VsCodeSetting.upsert_all(vs_code_settings) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/drop_vulnerabilities_without_finding_id.rb b/lib/gitlab/background_migration/drop_vulnerabilities_without_finding_id.rb new file mode 100644 index 00000000000..783bf0e2bda --- /dev/null +++ b/lib/gitlab/background_migration/drop_vulnerabilities_without_finding_id.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DropVulnerabilitiesWithoutFindingId < BatchedMigrationJob + operation_name :drop_vulnerabilities_without_finding_id + scope_to ->(relation) { relation.where(finding_id: nil) } + feature_category :vulnerability_management + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_workspaces_config_version3.rb b/lib/gitlab/background_migration/update_workspaces_config_version3.rb new file mode 100644 index 00000000000..8626f7f608d --- /dev/null +++ b/lib/gitlab/background_migration/update_workspaces_config_version3.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # No op on ce + class UpdateWorkspacesConfigVersion3 < BatchedMigrationJob + feature_category :remote_development + def perform; end + end + end +end + +Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion3.prepend_mod_with('Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion3') # rubocop:disable Layout/LineLength -- Injecting extension modules must be done on the last line of this file, outside of any class or module definitions diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 91994c2fa95..54cc63ab1a6 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -8,8 +8,6 @@ module Gitlab include EnforcesTwoFactorAuthentication include SessionsHelper - before_action :limit_session_time, if: -> { !current_user } - helper_method :can? end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb index 0d4de385f5e..99f4adbe317 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb @@ -10,6 +10,7 @@ module Gitlab @project = project @formatter = Gitlab::ImportFormatter.new @user_finder = UserFinder.new(project) + @mentions_converter = Gitlab::BitbucketServerImport::MentionsConverter.new(project.id) # Object should behave as a object so we can remove object.is_a?(Hash) check # This will be fixed in https://gitlab.com/gitlab-org/gitlab/-/issues/412328 @@ -19,10 +20,6 @@ module Gitlab def execute log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) - description = '' - description += author_line - description += object[:description] if object[:description] - attributes = { iid: object[:iid], title: object[:title], @@ -49,7 +46,19 @@ module Gitlab private - attr_reader :object, :project, :formatter, :user_finder + attr_reader :object, :project, :formatter, :user_finder, :mentions_converter + + def description + description = '' + description += author_line + description += object[:description] if object[:description] + + if Feature.enabled?(:bitbucket_server_convert_mentions_to_users, project.creator) + description = mentions_converter.convert(description) + end + + description + end def author_line return '' if user_finder.uid(object) diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb index d58f7cec8ff..19e5cdcbdc2 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb @@ -11,6 +11,7 @@ module Gitlab @project = project @user_finder = UserFinder.new(project) @formatter = Gitlab::ImportFormatter.new + @mentions_converter = Gitlab::BitbucketServerImport::MentionsConverter.new(project.id) @object = hash.with_indifferent_access end @@ -43,7 +44,7 @@ module Gitlab private - attr_reader :object, :project, :formatter, :user_finder + attr_reader :object, :project, :formatter, :user_finder, :mentions_converter def import_data_valid? project.import_data&.credentials && project.import_data&.data @@ -192,12 +193,18 @@ module Gitlab note = "*By #{comment.author_username} (#{comment.author_email})*\n\n" end + comment_note = if Feature.enabled?(:bitbucket_server_convert_mentions_to_users, project.creator) + mentions_converter.convert(comment.note) + else + comment.note + end + note += # Provide some context for replying if comment.parent_comment - "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}" + "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment_note}" else - comment.note + comment_note end { diff --git a/lib/gitlab/bitbucket_server_import/importers/users_importer.rb b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb index f8d0521afb2..156d89c2732 100644 --- a/lib/gitlab/bitbucket_server_import/importers/users_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb @@ -5,7 +5,7 @@ module Gitlab module Importers class UsersImporter include Loggable - include UserCaching + include UserFromMention BATCH_SIZE = 100 @@ -19,23 +19,26 @@ module Gitlab def execute log_info(import_stage: 'import_users', message: 'starting') - page = 1 + current = page_counter.current loop do log_info( import_stage: 'import_users', - message: "importing page #{page} using batch size #{BATCH_SIZE}" + message: "importing page #{current} using batch size #{BATCH_SIZE}" ) - users = client.users(project_key, page_offset: page, limit: BATCH_SIZE).to_a + users = client.users(project_key, page_offset: current, limit: BATCH_SIZE).to_a break if users.empty? cache_users(users) - page += 1 + current += 1 + page_counter.set(current) end + page_counter.expire! + log_info(import_stage: 'import_users', message: 'finished') end @@ -47,7 +50,7 @@ module Gitlab hash[cache_key] = user.email end - ::Gitlab::Cache::Import::Caching.write_multiple(users_hash) + cache_multiple(users_hash) end def client @@ -57,6 +60,10 @@ module Gitlab def project_key project.import_data.data['project_key'] end + + def page_counter + @page_counter ||= Gitlab::Import::PageCounter.new(project, :users, 'bitbucket-server-importer') + end end end end diff --git a/lib/gitlab/bitbucket_server_import/mentions_converter.rb b/lib/gitlab/bitbucket_server_import/mentions_converter.rb new file mode 100644 index 00000000000..8b1eeb6e007 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/mentions_converter.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class MentionsConverter + include UserFromMention + + MENTIONS_REGEX = User.reference_pattern + MENTION_PLACEHOLDER = '~GITLAB_MENTION_PLACEHOLDER~' + + attr_reader :project_id + + def initialize(project_id) + @project_id = project_id + end + + def convert(text) + replace_mentions(text.dup) + end + + private + + def replace_mentions(text) + mentions = text.scan(MENTIONS_REGEX).flatten + altered_mentions = [] + + mentions.each do |mention| + user = user_from_cache(mention) + + if user + altered_mentions << ["@#{mention}", "#{MENTION_PLACEHOLDER}#{user.username}"] + next + end + + altered_mentions << ["@#{mention}", "`#{MENTION_PLACEHOLDER}#{mention}`"] + end + + altered_mentions.each do |original_mention, altered_mention| + text.sub!(original_mention, altered_mention) + end + + text.gsub(MENTION_PLACEHOLDER, '@') + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/user_caching.rb b/lib/gitlab/bitbucket_server_import/user_caching.rb deleted file mode 100644 index 0f0169122c5..00000000000 --- a/lib/gitlab/bitbucket_server_import/user_caching.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BitbucketServerImport - module UserCaching - SOURCE_USER_CACHE_KEY = 'bitbucket_server/project/%s/source/username/%s' - - def source_user_cache_key(project_id, username) - format(SOURCE_USER_CACHE_KEY, project_id, username) - end - end - end -end diff --git a/lib/gitlab/bitbucket_server_import/user_from_mention.rb b/lib/gitlab/bitbucket_server_import/user_from_mention.rb new file mode 100644 index 00000000000..907db245760 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/user_from_mention.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module UserFromMention + SOURCE_USER_CACHE_KEY = 'bitbucket_server/project/%s/source/username/%s' + + def user_from_cache(mention) + cached_email = read(mention) + + return unless cached_email + + find_user(cached_email) + end + + def cache_multiple(hash) + ::Gitlab::Cache::Import::Caching.write_multiple(hash, timeout: timeout) + end + + def source_user_cache_key(project_id, username) + format(SOURCE_USER_CACHE_KEY, project_id, username) + end + + private + + def read(mention) + ::Gitlab::Cache::Import::Caching.read(source_user_cache_key(project_id, mention)) + end + + def find_user(email) + User.find_by_any_email(email, confirmed: true) + end + + def timeout + ::Gitlab::Cache::Import::Caching::LONGER_TIMEOUT + end + end + end +end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index e81a90831f7..f3251d47e7a 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -239,6 +239,48 @@ module Gitlab end end + # Adds a value to a list. + # + # raw_key - The key of the list to add to. + # value - The field value to add to the list. + # timeout - The new timeout of the key. + # limit - The maximum number of members in the set. Older members will be trimmed to this limit. + def self.list_add(raw_key, value, timeout: TIMEOUT, limit: nil) + validate_redis_value!(value) + + key = cache_key_for(raw_key) + + with_redis do |redis| + redis.multi do |m| + m.rpush(key, value) + m.ltrim(key, -limit, -1) if limit + m.expire(key, timeout) + end + end + end + + # Returns the values of the given list. + # + # raw_key - The key of the list. + def self.values_from_list(raw_key) + key = cache_key_for(raw_key) + + with_redis do |redis| + redis.lrange(key, 0, -1) + end + end + + # Deletes a key + # + # raw_key - Key name + def self.del(raw_key) + key = cache_key_for(raw_key) + + with_redis do |redis| + redis.del(key) + end + end + def self.cache_key_for(raw_key) "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 8b503290e6e..999f41ff46f 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,7 +6,9 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors, keyword_init: true) do + Result = Struct.new( + :when, :start_in, :allow_failure, :variables, :needs, :errors, :auto_cancel, keyword_init: true + ) do def build_attributes needs_job = needs&.dig(:job) { @@ -37,7 +39,8 @@ module Gitlab start_in: matched_rule.attributes[:start_in], allow_failure: matched_rule.attributes[:allow_failure], variables: matched_rule.attributes[:variables], - needs: matched_rule.attributes[:needs] + needs: matched_rule.attributes[:needs], + auto_cancel: matched_rule.attributes[:auto_cancel] ) else Result.new(when: 'never') diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 0b322fd433c..d19140851f5 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -58,7 +58,8 @@ module Gitlab description: 'List of evaluable Rules to determine job inclusion.', inherit: false, metadata: { - allowed_when: %w[on_success on_failure always never manual delayed].freeze + allowed_when: %w[on_success on_failure always never manual delayed].freeze, + allowed_keys: %i[if changes exists when start_in allow_failure variables needs].freeze } entry :variables, ::Gitlab::Ci::Config::Entry::Variables, diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 1e7f6056a65..81e67592c29 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -4,14 +4,16 @@ module Gitlab module Ci class Config module Entry + # A rule is a condition that is evaluated before a job is executed. + # Until we find a better solution in https://gitlab.com/gitlab-org/gitlab/-/issues/436473, + # these two metadata parameters need to be passed to `Entry::Rules`: + # - `allowed_when`: a list of allowed values for the `when` keyword. + # - `allowed_keys`: a list of allowed keys for each rule. class Rules::Rule < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables needs].freeze - ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze - attributes :if, :exists, :when, :start_in, :allow_failure entry :changes, Entry::Rules::Rule::Changes, @@ -25,10 +27,12 @@ module Gitlab metadata: { allowed_needs: %i[job] }, inherit: false + entry :auto_cancel, Entry::AutoCancel, + description: 'Auto-cancel configuration for the pipeline.' + validations do validates :config, presence: true validates :config, type: { with: Hash } - validates :config, allowed_keys: ALLOWED_KEYS validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? validates :start_in, presence: true, if: :specifies_delay? validates :start_in, duration: { limit: '1 week' }, if: :specifies_delay? @@ -36,15 +40,22 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true validates :exists, array_of_strings: true, length: { maximum: 50 } - validates :when, allowed_values: { in: ALLOWED_WHEN } validates :allow_failure, boolean: true end validate do + # This validation replaces the old `validates :when, allowed_values: { in: ALLOWED_WHEN }` validation. + # In https://gitlab.com/gitlab-org/gitlab/-/issues/436473, we'll remove this custom validation. validates_with Gitlab::Config::Entry::Validators::AllowedValuesValidator, attributes: %i[when], allow_nil: true, in: opt(:allowed_when) + + # This validation replaces the old `validates :config, allowed_keys: ALLOWED_KEYS` validation. + # In https://gitlab.com/gitlab-org/gitlab/-/issues/436473, we'll remove this custom validation. + validates_with Gitlab::Config::Entry::Validators::AllowedKeysValidator, + attributes: %i[config], + in: opt(:allowed_keys) end end @@ -52,7 +63,8 @@ module Gitlab config.merge( changes: (changes_value if changes_defined?), variables: (variables_value if variables_defined?), - needs: (needs_value if needs_defined?) + needs: (needs_value if needs_defined?), + auto_cancel: (auto_cancel_value if auto_cancel_defined?) ).compact end diff --git a/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json index a31374650e6..1098da0111a 100644 --- a/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json +++ b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json @@ -10,6 +10,11 @@ "type": "string", "minLength": 1, "maxLength": 64 + }, + "user": { + "type": "string", + "minLength": 1, + "maxLength": 255 } }, "additionalProperties": false diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 5b81c74fe4d..b201989a9f7 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -19,9 +19,14 @@ module Gitlab validates :name, allow_nil: true, length: { minimum: 1, maximum: 255 } end + # `start_in`, `allow_failure`, and `needs` should not be allowed but we can't break this behavior now. + # More information: https://gitlab.com/gitlab-org/gitlab/-/issues/436473 entry :rules, Entry::Rules, description: 'List of evaluable Rules to determine Pipeline status.', - metadata: { allowed_when: %w[always never] } + metadata: { + allowed_when: %w[always never].freeze, + allowed_keys: %i[if changes exists when start_in allow_failure variables needs auto_cancel].freeze + } entry :auto_cancel, Entry::AutoCancel, description: 'Auto-cancel configuration for this pipeline.' diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 0a524fdba66..cbbea3c7c12 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -11,13 +11,16 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize - attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config - attr_reader :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config, :parallel_requests, + :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes attr_accessor :total_file_size_in_bytes delegate :instrument, to: :logger + # We try to keep the number of parallel HTTP requests to a minimum to avoid overloading IO. + MAX_PARALLEL_REMOTE_REQUESTS = 2 + def initialize( project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, pipeline_config: nil, logger: nil @@ -30,6 +33,7 @@ module Gitlab @variables = variables || Ci::Variables::Collection.new @pipeline_config = pipeline_config @expandset = [] + @parallel_requests = [] @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes @@ -65,6 +69,7 @@ module Gitlab ctx.logger = logger ctx.max_includes = max_includes ctx.max_total_yaml_size_bytes = max_total_yaml_size_bytes + ctx.parallel_requests = parallel_requests end end @@ -76,6 +81,16 @@ module Gitlab raise TimeoutError if execution_expired? end + def execute_remote_parallel_request(lazy_response) + parallel_requests.delete_if(&:complete?) + + # We are "assuming" that the first request in the queue is the first one to complete. + # This is good enough approximation. + parallel_requests.first&.wait unless parallel_requests.size < MAX_PARALLEL_REMOTE_REQUESTS + + parallel_requests << lazy_response.execute + end + def sentry_payload { user: user.inspect, @@ -106,7 +121,8 @@ module Gitlab protected - attr_writer :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + attr_writer :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes, + :parallel_requests private diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 03063e76dde..ab44b424d8d 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -18,7 +18,12 @@ module Gitlab def content return unless component_result.success? - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('cicd_component_usage', values: context.user.id) + if context.user.present? + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event( + 'cicd_component_usage', + values: context.user.id + ) + end component_payload.fetch(:content) end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index fc90b497f85..266901811f6 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -54,10 +54,12 @@ module Gitlab private def fetch_async_content - return if ::Feature.disabled?(:ci_parallel_remote_includes, context.project) + return unless YamlProcessor::FeatureFlags.enabled?(:ci_parallel_remote_includes) - # It starts fetching the remote content in a separate thread and returns a promise immediately. - Gitlab::HTTP.get(location, async: true).execute + # It starts fetching the remote content in a separate thread and returns a lazy_response immediately. + Gitlab::HTTP.get(location, async: true).tap do |lazy_response| + context.execute_remote_parallel_request(lazy_response) + end end strong_memoize_attr :fetch_async_content diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index 79c1c14dc4e..62cd322e141 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -58,6 +58,7 @@ module Gitlab def parse_components data['components']&.each_with_index do |component_data, index| + properties = component_data['properties'] component = ::Gitlab::Ci::Reports::Sbom::Component.new( type: component_data['type'], name: component_data['name'], @@ -65,6 +66,7 @@ module Gitlab version: component_data['version'] ) + component.properties = CyclonedxProperties.parse_trivy_source(properties) if properties report.add_component(component) if component.ingestible? rescue ::Sbom::PackageUrl::InvalidPackageUrl report.add_error("/components/#{index}/purl is invalid") diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb index 35548358c57..7069e784934 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -5,7 +5,7 @@ module Gitlab module Parsers module Sbom # Parses GitLab CycloneDX metadata properties which are defined by the taxonomy at - # https://gitlab.com/gitlab-org/security-products/gitlab-cyclonedx-property-taxonomy + # https://docs.gitlab.com/ee/development/sec/cyclonedx_property_taxonomy.html # # This parser knows how to process schema version 1 and will not attempt to parse # later versions. Each source type has it's own namespace in the property schema, @@ -14,10 +14,13 @@ module Gitlab class CyclonedxProperties SUPPORTED_SCHEMA_VERSION = '1' GITLAB_PREFIX = 'gitlab:' + AQUASECURITY_PREFIX = 'aquasecurity:' SOURCE_PARSERS = { 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning, - 'container_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::ContainerScanning + 'container_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::ContainerScanning, + 'trivy' => ::Gitlab::Ci::Parsers::Sbom::Source::Trivy }.freeze + SUPPORTED_PROPERTIES = %w[ meta:schema_version dependency_scanning:category @@ -29,12 +32,26 @@ module Gitlab container_scanning:image:tag container_scanning:operating_system:name container_scanning:operating_system:version + trivy:PkgID + trivy:PkgType + trivy:SrcName + trivy:SrcVersion + trivy:SrcRelease + trivy:SrcEpoch + trivy:Modularitylabel + trivy:FilePath + trivy:LayerDigest + trivy:LayerDiffID ].freeze def self.parse_source(...) new(...).parse_source end + def self.parse_trivy_source(...) + new(...).parse_trivy_source + end + def initialize(properties) @properties = properties end @@ -46,6 +63,12 @@ module Gitlab source end + def parse_trivy_source + return unless properties.present? + + source + end + private attr_reader :properties @@ -61,11 +84,15 @@ module Gitlab # The specification permits the name or value to be absent. return unless name.present? && value.present? - return unless name.start_with?(GITLAB_PREFIX) - namespaced_name = name.delete_prefix(GITLAB_PREFIX) + namespaced_name = + if name.start_with?(GITLAB_PREFIX) + name.delete_prefix(GITLAB_PREFIX) + elsif name.start_with?(AQUASECURITY_PREFIX) + name.delete_prefix(AQUASECURITY_PREFIX) + end - return unless SUPPORTED_PROPERTIES.include?(namespaced_name) + return unless namespaced_name && SUPPORTED_PROPERTIES.include?(namespaced_name) parse_name_value_pair(namespaced_name, value, data) end diff --git a/lib/gitlab/ci/parsers/sbom/source/trivy.rb b/lib/gitlab/ci/parsers/sbom/source/trivy.rb new file mode 100644 index 00000000000..0218b19e931 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/trivy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class Trivy < BaseSource + private + + def type + :trivy + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index be6c6c2558b..ede0f62ea51 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -20,6 +20,7 @@ module Gitlab end def parse! + sanitize_json_data set_report_version return report_data unless valid? @@ -43,6 +44,14 @@ module Gitlab attr_reader :json_data, :report, :validate, :project + # PostgreSQL can not save texts with unicode null character + # that's why we are escaping that character. + def sanitize_json_data + return unless json_data.gsub!('\u0000', '\\\\\u0000') + + report.add_warning('Parsing', 'Report artifact contained unicode null characters which are escaped during the ingestion.') + end + def valid? return true unless validate @@ -123,7 +132,6 @@ module Gitlab uuid: uuid, report_type: report.type, name: finding_name(data, identifiers, location), - compare_key: data['cve'] || '', location: location, evidence: evidence, severity: ::Enums::Vulnerability.parse_severity_level(data['severity']), diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index cc3aa33e93b..7e871732c20 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -62,7 +62,7 @@ module Gitlab end def before_sha - self[:before_sha] || checkout_sha || Gitlab::Git::BLANK_SHA + self[:before_sha] || checkout_sha || Gitlab::Git::SHA1_BLANK_SHA end def protected_ref? diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index ab37eb93f18..5fdba860b0e 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -13,13 +13,9 @@ module Gitlab return if workflow_passed? - if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) - drop_reason = :filtered_by_workflow_rules - end - error( 'Pipeline filtered out by workflow rules.', - drop_reason: drop_reason + drop_reason: :filtered_by_workflow_rules ) end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 0e55928ff80..5dfe918042e 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -45,7 +45,7 @@ module Gitlab else command.increment_pipeline_failure_reason_counter(drop_reason) - pipeline.set_failed(drop_reason) if Feature.enabled?(:always_set_pipeline_failure_reason, command.project) + pipeline.set_failed(drop_reason) end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index f73addcd098..e9097182262 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,14 +18,10 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) - drop_reason = :filtered_by_rules - end - return error( 'Pipeline will not run for the selected trigger. ' \ 'The rules configuration prevented any jobs from being added to the pipeline.', - drop_reason: drop_reason + drop_reason: :filtered_by_rules ) end diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index 3ac910da752..8e6426be679 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -35,7 +35,10 @@ module Gitlab end def set_auto_cancel - auto_cancel = @command.yaml_processor_result.workflow_auto_cancel + auto_cancel_from_config = @command.yaml_processor_result.workflow_auto_cancel || {} + auto_cancel_from_rules = @command.workflow_rules_result&.auto_cancel || {} + + auto_cancel = auto_cancel_from_config.merge(auto_cancel_from_rules) return if auto_cancel.blank? diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 59816e75b2c..1a3f689c1d7 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -8,12 +8,14 @@ module Gitlab include Gitlab::Utils::StrongMemoize attr_reader :component_type, :version, :path + attr_accessor :properties - def initialize(type:, name:, purl:, version:) + def initialize(type:, name:, purl:, version:, properties: nil) @component_type = type @name = name @raw_purl = purl @version = version + @properties = properties end def <=>(other) diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index fa8494483d3..fbca1e674d1 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -7,7 +7,6 @@ module Gitlab class Finding include ::VulnerabilityFindingHelpers - attr_reader :compare_key attr_reader :confidence attr_reader :identifiers attr_reader :flags @@ -34,10 +33,7 @@ module Gitlab delegate :file_path, :start_line, :end_line, to: :location - alias_method :cve, :compare_key - - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil, cvss: []) # rubocop:disable Metrics/ParameterLists - @compare_key = compare_key + def initialize(identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil, cvss: []) # rubocop:disable Metrics/ParameterLists @confidence = confidence @identifiers = identifiers @flags = flags @@ -65,7 +61,6 @@ module Gitlab def to_hash %i[ - compare_key confidence identifiers flags @@ -84,7 +79,6 @@ module Gitlab details signatures description - cve solution ].index_with do |key| public_send(key) # rubocop:disable GitlabSecurity/PublicSend @@ -141,7 +135,7 @@ module Gitlab def <=>(other) if severity == other.severity - compare_key <=> other.compare_key + uuid <=> other.uuid else ::Enums::Vulnerability.severity_levels[other.severity] <=> ::Enums::Vulnerability.severity_levels[severity] @@ -200,7 +194,7 @@ module Gitlab private def generate_project_fingerprint - Digest::SHA1.hexdigest(compare_key) + Digest::SHA1.hexdigest(uuid.to_s) end def location_fingerprints diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index a5cddf5d2d7..6f8bed32796 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 0a899f3bb74..52367cfe97d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 87a7f79c0ce..06dc91a8bbc 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.76.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 1ed4cd86e82..4b60298353d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: dast: stage: dast image: - name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: true @@ -30,3 +30,10 @@ dast: artifacts: reports: dast: gl-dast-report.json + rules: + - if: $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_IMAGE_SUFFIX: "-fips" + - if: $CI_GITLAB_FIPS_MODE != "true" + variables: + DAST_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml index c75ff2e9ff8..8043b6a95cc 100644 --- a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -18,9 +18,16 @@ variables: validation: stage: dast image: - name: "$CI_TEMPLATE_REGISTRY_HOST/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" + name: "$CI_TEMPLATE_REGISTRY_HOST/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: false script: - ~/validate.sh + rules: + - if: $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_IMAGE_SUFFIX: "-fips" + - if: $CI_GITLAB_FIPS_MODE != "true" + variables: + DAST_IMAGE_SUFFIX: "" diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/job_artifact_object.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/job_artifact_object.rb new file mode 100644 index 00000000000..61e7c6c43a6 --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/job_artifact_object.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + module OrphanJobArtifactFinalObjects + class JobArtifactObject + include Gitlab::Utils::StrongMemoize + + attr_reader :path, :size + + def initialize(fog_file, bucket_prefix: nil) + @fog_file = fog_file + @path = fog_file.key + @size = fog_file.content_length + @bucket_prefix = bucket_prefix + end + + def in_final_location? + path.include?('/@final/') + end + + def orphan? + !job_artifact_record_exists? && !pending_direct_upload? + end + + def delete + fog_file.destroy + end + + private + + attr_reader :fog_file, :bucket_prefix + + def job_artifact_record_exists? + ::Ci::JobArtifact.exists?(file_final_path: path_without_bucket_prefix) # rubocop:disable CodeReuse/ActiveRecord -- too simple and specific for this usecase to be its own AR method + end + + def pending_direct_upload? + ::ObjectStorage::PendingDirectUpload.exists?(:artifacts, path_without_bucket_prefix) # rubocop:disable CodeReuse/ActiveRecord -- `exists?` here is not the same as the AR method + end + + def path_without_bucket_prefix + # `path` contains the fog file's key. It is the object path relative to the artifacts bucket, for example: + # aa/bb/abc123/@final/12/34/def12345 + # + # But if the instance is configured to only use a single bucket combined with bucket prefixes, + # for example if the `bucket_prefix` is "my/artifacts", the `path` would then look like: + # my/artifacts/aa/bb/abc123/@final/12/34/def12345 + # + # For `orphan?` to function properly, we need to strip the bucket_prefix + # off of the `path` because we need this to match the correct job artifact record by + # its `file_final_path` column, or the pending direct upload redis entry, which both contains + # the object's path without `bucket_prefix`. + # + # If bucket_prefix is not present, this will just return the original path. + Pathname.new(path).relative_path_from(bucket_prefix.to_s).to_s + end + strong_memoize_attr :path_without_bucket_prefix + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/aws.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/aws.rb new file mode 100644 index 00000000000..7fedd8f4306 --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/aws.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + module OrphanJobArtifactFinalObjects + module Paginators + class Aws < BasePaginator + def page_marker_filter_key + :marker + end + + def max_results_filter_key + :max_keys + end + + def last_page?(batch) + batch.empty? + end + + def get_next_marker(batch) + batch.last.key + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/base_paginator.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/base_paginator.rb new file mode 100644 index 00000000000..7bc7f9c2661 --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/base_paginator.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + module OrphanJobArtifactFinalObjects + module Paginators + class BasePaginator + BATCH_SIZE = Rails.env.development? ? 5 : 200 + + def initialize(bucket_prefix: nil) + @bucket_prefix = bucket_prefix + end + + def filters(marker) + { + page_marker_filter_key => marker, + max_results_filter_key => BATCH_SIZE, + prefix: bucket_prefix + } + end + + def last_page?(batch) + # Fog providers have different indicators of last page, so we want to delegate this + # knowledge to the specific provider implementation. + raise NotImplementedError, "Subclasses must define `last_page?(batch)` instance method" + end + + def get_next_marker(batch) + # Fog providers have different ways to get the next marker, so we want to delegate this + # knowledge to the specific provider implementation. + raise NotImplementedError, "Subclasses must define `get_next_marker(batch)` instance method" + end + + private + + attr_reader :bucket_prefix + + def page_marker_filter_key + raise NotImplementedError, "Subclasses must define `page_marker_key` instance method" + end + + def max_results_filter_key + raise NotImplementedError, "Subclasses must define `max_results_filter_key` instance method" + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/google.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/google.rb new file mode 100644 index 00000000000..9b0da9910cd --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects/paginators/google.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + module OrphanJobArtifactFinalObjects + module Paginators + class Google < BasePaginator + def filters(marker) + pattern = [bucket_prefix, '*/*/*/@final/**'].compact.join('/') + super.merge(match_glob: pattern) + end + + def page_marker_filter_key + :page_token + end + + def max_results_filter_key + :max_results + end + + def last_page?(batch) + batch.next_page_token.nil? + end + + def get_next_marker(batch) + batch.next_page_token + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_final_objects_cleaner.rb b/lib/gitlab/cleanup/orphan_job_artifact_final_objects_cleaner.rb new file mode 100644 index 00000000000..4726d68e024 --- /dev/null +++ b/lib/gitlab/cleanup/orphan_job_artifact_final_objects_cleaner.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class OrphanJobArtifactFinalObjectsCleaner + include Gitlab::Utils::StrongMemoize + + UnsupportedProviderError = Class.new(StandardError) + + PAGINATORS = { + google: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Google, + aws: Gitlab::Cleanup::OrphanJobArtifactFinalObjects::Paginators::Aws + }.freeze + + LAST_PAGE_MARKER_REDIS_KEY = 'orphan-job-artifact-objects-cleanup-last-page-marker' + + def initialize(provider: nil, dry_run: true, force_restart: false, logger: Gitlab::AppLogger) + @paginator = determine_paginator!(provider) + @dry_run = dry_run + @force_restart = force_restart + @logger = logger + end + + def run! + log_info('Looking for orphan job artifact objects under the `@final` directories') + + each_final_object do |object| + next unless object.orphan? + + object.delete unless dry_run + log_info("Delete #{object.path} (#{object.size} bytes)") + end + + log_info("Done.") + end + + private + + attr_reader :paginator, :dry_run, :force_restart, :logger + + def determine_paginator!(provided_provider) + # provider can be nil if user didn't specify it when running the clean up task. + # In this case, we automatically determine the provider based on the object storage configuration. + provider = provided_provider + provider ||= configuration.connection.provider + klass = PAGINATORS.fetch(provider.downcase.to_sym) + klass.new(bucket_prefix: bucket_prefix) + rescue KeyError + msg = if provided_provider.present? + "The provided provider is unsupported. Please select from #{PAGINATORS.keys.join(', ')}." + else + <<-MSG.strip_heredoc + The provider found in the object storage configuration is unsupported. + Please re-run the task and specify a provider from #{PAGINATORS.keys.join(', ')}, + whichever is compatible with your provider's object storage API." + MSG + end + + raise UnsupportedProviderError, msg + end + + def each_final_object + each_batch do |files| + files.each_file_this_page do |fog_file| + object = ::Gitlab::Cleanup::OrphanJobArtifactFinalObjects::JobArtifactObject.new( + fog_file, + bucket_prefix: bucket_prefix + ) + + # We still need to check here if the object is in the final location because + # if the provider does not support filtering objects by glob pattern, we will + # then receive all job artifact objects here, even the ones not in the @final directory. + yield object if object.in_final_location? + end + end + end + + def each_batch + next_marker = resume_from_last_page_marker + + loop do + batch = fetch_batch(next_marker) + yield batch + + break if paginator.last_page?(batch) + + next_marker = paginator.get_next_marker(batch) + save_last_page_marker(next_marker) + end + + clear_last_page_marker + end + + def fetch_batch(marker) + page_name = marker ? "marker: #{marker}" : "first page" + log_info("Loading page (#{page_name})") + + # We are using files.all instead of files.each because we want to track the + # current page token so that we can resume from it if ever the task is abruptly interrupted. + artifacts_directory.files.all( + paginator.filters(marker) + ) + end + + def resume_from_last_page_marker + if force_restart + log_info("Force restarted. Will not resume from last known page marker.") + nil + else + get_last_page_marker + end + end + + def get_last_page_marker + Gitlab::Redis::SharedState.with do |redis| + marker = redis.get(LAST_PAGE_MARKER_REDIS_KEY) + log_info("Resuming from last page marker: #{marker}") if marker + marker + end + end + + def save_last_page_marker(marker) + Gitlab::Redis::SharedState.with do |redis| + # Set TTL to 1 day (86400 seconds) + redis.set(LAST_PAGE_MARKER_REDIS_KEY, marker, ex: 86400) + end + end + + def clear_last_page_marker + Gitlab::Redis::SharedState.with do |redis| + redis.del(LAST_PAGE_MARKER_REDIS_KEY) + end + end + + def connection + ::Fog::Storage.new(configuration['connection'].symbolize_keys) + end + + def configuration + Gitlab.config.artifacts.object_store + end + + def bucket + configuration.remote_directory + end + + def bucket_prefix + configuration.bucket_prefix + end + + def artifacts_directory + connection.directories.new(key: bucket) + end + strong_memoize_attr :artifacts_directory + + def log_info(msg) + logger.info("#{'[DRY RUN] ' if dry_run}#{msg}") + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 918f723cd60..7c376893156 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -122,7 +122,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def project_id - @project_id ||= Project.where_full_path_in([full_path], use_includes: false).pluck(:id) + @project_id ||= Project.where_full_path_in([full_path], preload_routes: false).pluck(:id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 5c4899da11f..64e0478734b 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -12,22 +12,18 @@ module Gitlab end def current_application_settings - Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } + Gitlab::SafeRequestStore.fetch(:current_application_settings) { Gitlab::ApplicationSettingFetcher.current_application_settings } end def current_application_settings? - Gitlab::SafeRequestStore.exist?(:current_application_settings) || ::ApplicationSetting.current.present? + Gitlab::SafeRequestStore.exist?(:current_application_settings) || Gitlab::ApplicationSettingFetcher.current_application_settings? end def expire_current_application_settings - ::ApplicationSetting.expire + Gitlab::ApplicationSettingFetcher.expire_current_application_settings Gitlab::SafeRequestStore.delete(:current_application_settings) end - def clear_in_memory_application_settings! - @in_memory_application_settings = nil - end - def method_missing(name, *args, **kwargs, &block) current_application_settings.send(name, *args, **kwargs, &block) # rubocop:disable GitlabSecurity/PublicSend end @@ -35,66 +31,6 @@ module Gitlab def respond_to_missing?(name, include_private = false) current_application_settings.respond_to?(name, include_private) || super end - - private - - def ensure_application_settings! - cached_application_settings || uncached_application_settings - end - - def cached_application_settings - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - - begin - ::ApplicationSetting.cached - rescue StandardError - # In case Redis isn't running - # or the Redis UNIX socket file is not available - # or the DB is not running (we use migrations in the cache key) - end - end - - def uncached_application_settings - return fake_application_settings if Gitlab::Runtime.rake? && !connect_to_db? - - current_settings = ::ApplicationSetting.current - # If there are pending migrations, it's possible there are columns that - # need to be added to the application settings. To prevent Rake tasks - # and other callers from failing, use any loaded settings and return - # defaults for missing columns. - if Gitlab::Runtime.rake? && ::ApplicationSetting.connection.migration_context.needs_migration? - db_attributes = current_settings&.attributes || {} - fake_application_settings(db_attributes) - elsif current_settings.present? - current_settings - else - ::ApplicationSetting.create_from_defaults - end - rescue ::ApplicationSetting::Recursion - in_memory_application_settings - end - - def fake_application_settings(attributes = {}) - Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) - end - - def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults - end - - def connect_to_db? - # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised - active_db_connection = begin - ::ApplicationSetting.connection.active? - rescue StandardError - false - end - - active_db_connection && - ApplicationSetting.database.cached_table_exists? - rescue ActiveRecord::NoDatabaseError - false - end end end end diff --git a/lib/gitlab/database/decomposition/migrate.rb b/lib/gitlab/database/decomposition/migrate.rb index b6ca5adf857..812e28c573f 100644 --- a/lib/gitlab/database/decomposition/migrate.rb +++ b/lib/gitlab/database/decomposition/migrate.rb @@ -52,7 +52,7 @@ module Gitlab ApplicationRecord.connection.execute( ApplicationRecord.sanitize_sql([ TABLE_SIZE_QUERY, - { table_catalog: main_config.dig(:activerecord, :database) } + { table_catalog: main_database_name } ]) ).first["total"].to_f end @@ -86,7 +86,7 @@ module Gitlab end def ci_database_connect_ok? - _, status = with_transient_pg_env(ci_config[:pg_env]) do + _, status = with_transient_pg_env(ci_config.pg_env_variables) do psql_args = ["--dbname=#{ci_database_name}", "-tAc", "select 1"] Open3.capture2e('psql', *psql_args) @@ -94,7 +94,7 @@ module Gitlab unless status.success? raise MigrateError, - "Can't connect to database '#{ci_database_name} on host '#{ci_config[:pg_env]['PGHOST']}'. " \ + "Can't connect to database '#{ci_database_name} on host '#{ci_config.pg_env_variables['PGHOST']}'. " \ "Ensure the database has been created." end @@ -107,7 +107,7 @@ module Gitlab { table_catalog: ci_database_name } ]) - output, status = with_transient_pg_env(ci_config[:pg_env]) do + output, status = with_transient_pg_env(ci_config.pg_env_variables) do psql_args = ["--dbname=#{ci_database_name}", "-tAc", sql] Open3.capture2e('psql', *psql_args) @@ -149,7 +149,7 @@ module Gitlab end def import_dump_to_ci_db - with_transient_pg_env(ci_config[:pg_env]) do + with_transient_pg_env(ci_config.pg_env_variables) do restore_args = ["--jobs=4", "--dbname=#{ci_database_name}"] Open3.capture2e('pg_restore', *restore_args, @backup_location) @@ -157,23 +157,27 @@ module Gitlab end def dump_main_db - with_transient_pg_env(main_config[:pg_env]) do + with_transient_pg_env(main_config.pg_env_variables) do args = ['--format=d', '--jobs=4', "--file=#{@backup_location}"] - Open3.capture2e('pg_dump', *args, main_config.dig(:activerecord, :database)) + Open3.capture2e('pg_dump', *args, main_database_name) end end def main_config - @main_config ||= ::Backup::DatabaseModel.new('main').config + @main_config ||= ::Backup::DatabaseConfiguration.new('main') end def ci_config - @ci_config ||= ::Backup::DatabaseModel.new('ci').config + @ci_config ||= ::Backup::DatabaseConfiguration.new('ci') + end + + def main_database_name + main_config.activerecord_configuration.database end def ci_database_name - @ci_database_name ||= "#{main_config.dig(:activerecord, :database)}_ci" + "#{main_config.activerecord_configuration.database}_ci" end end end diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb index 4ef392a4e44..d963d6f288e 100644 --- a/lib/gitlab/database/dictionary.rb +++ b/lib/gitlab/database/dictionary.rb @@ -3,6 +3,8 @@ module Gitlab module Database class Dictionary + ALL_SCOPES = ['', 'views', 'deleted_tables'].freeze + def self.entries(scope = '') @entries ||= {} @entries[scope] ||= Dir.glob(dictionary_path_globs(scope)).map do |file_path| @@ -12,6 +14,15 @@ module Gitlab end end + def self.any_entry(name) + ALL_SCOPES.each do |scope| + e = entry(name, scope) + return e if e + end + + nil + end + def self.entry(name, scope = '') entries(scope).find do |entry| entry.key_name == name @@ -69,6 +80,10 @@ module Gitlab data['classes'] end + def allow_cross_to_schemas(type) + data["allow_cross_#{type}"].to_a.map(&:to_sym) + end + def schema?(schema_name) gitlab_schema == schema_name.to_s end diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb index b7ec3dfc893..36e586313a5 100644 --- a/lib/gitlab/database/gitlab_schema_info.rb +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -2,11 +2,6 @@ module Gitlab module Database - GitlabSchemaInfoAllowCross = Struct.new( - :specific_tables, - keyword_init: true - ) - GitlabSchemaInfo = Struct.new( :name, :description, @@ -20,9 +15,12 @@ module Gitlab def initialize(*) super self.name = name.to_sym - self.allow_cross_joins = convert_array_to_hash(allow_cross_joins) - self.allow_cross_transactions = convert_array_to_hash(allow_cross_transactions) - self.allow_cross_foreign_keys = convert_array_to_hash(allow_cross_foreign_keys) + self.allow_cross_joins = add_table_specific_allows( + :joins, convert_array_to_hash(allow_cross_joins)) + self.allow_cross_transactions = add_table_specific_allows( + :transactions, convert_array_to_hash(allow_cross_transactions)) + self.allow_cross_foreign_keys = add_table_specific_allows( + :foreign_keys, convert_array_to_hash(allow_cross_foreign_keys)) end def self.load_file(yaml_file) @@ -31,35 +29,37 @@ module Gitlab end def allow_cross_joins?(table_schemas, all_tables) - allowed_schemas = allow_cross_joins || {} - - allowed_for?(allowed_schemas, table_schemas, all_tables) + allowed_for?(allow_cross_joins, table_schemas, all_tables) end def allow_cross_transactions?(table_schemas, all_tables) - allowed_schemas = allow_cross_transactions || {} - - allowed_for?(allowed_schemas, table_schemas, all_tables) + allowed_for?(allow_cross_transactions, table_schemas, all_tables) end def allow_cross_foreign_keys?(table_schemas, all_tables) - allowed_schemas = allow_cross_foreign_keys || {} - - allowed_for?(allowed_schemas, table_schemas, all_tables) + allowed_for?(allow_cross_foreign_keys, table_schemas, all_tables) end private def allowed_for?(allowed_schemas, table_schemas, all_tables) + # Take all the schemas in the query and remove the current schema and all the allowed schemas. If there is + # anything left then it's not allowed. Then we even if there is nothing left we continue to verify + # `specific_tables` used in the allowed schemas. denied_schemas = table_schemas - [name] denied_schemas -= allowed_schemas.keys return false unless denied_schemas.empty? + # Additional validation for specific_tables. We should validate that if `specific_tables` is set then we will + # need all the tables to be in the the allowed specific_tables all_tables.all? do |table| table_schema = ::Gitlab::Database::GitlabSchema.table_schema!(table) allowed_tables = allowed_schemas[table_schema] - allowed_tables.nil? || allowed_tables.specific_tables.include?(table) + # If specific tables key is nil? (not present) then we assume all tables are allowed and return true Otherwise + # we check every table in the current query is in specific_tables list + allowed_tables.nil? || + allowed_tables[:specific_tables].include?(table) end end @@ -72,7 +72,7 @@ module Gitlab # # To: # { :schema_a => nil, - # :schema_b => { specific_tables : [:table_b_of_schema_b, :table_c_of_schema_b] } + # :schema_b => { specific_tables : ['table_b_of_schema_b', 'table_c_of_schema_b'] } # } # def convert_array_to_hash(subject) @@ -81,15 +81,58 @@ module Gitlab subject&.each do |item| if item.is_a?(Hash) item.each do |key, value| - result[key.to_sym] = GitlabSchemaInfoAllowCross.new(value || {}) + result[key.to_sym] = { specific_tables: value[:specific_tables].to_set } end else result[item.to_sym] = nil end end + result + end + + # This method loops over all the `db/docs` files for every table and injects any + # allow_cross_joins/allow_cross_transactions/allow_cross_foreign_keys into the specific_tables lists for the + # current schema. + def add_table_specific_allows(type, schema_allows) + result = schema_allows + all_table_allows(type).each do |schema_from, tables| + # Preserve the meaning of `nil` as defined in convert_array_to_hash as a nil value means that we allow all + # tables + next if result.key?(schema_from) && result[schema_from].nil? + + # Now we add the table to the specific_tables list because this table specifies it is allowed in this schema + result[schema_from] ||= { specific_tables: Set.new } + result[schema_from][:specific_tables] += tables + end result.freeze end + + # For the given type we iterate over all db/docs files build a Hash like: + # + # { + # gitlab_main_cell: ['table_a', 'table_b'] + # } + # + # This specifies that in the `gitlab_main_cell` schema the 'table_a` and `table_b` tables are allowing cross + # queries with the current schema + def all_table_allows(type) + @all_table_allows ||= {} + @all_table_allows[type] ||= begin + result = {} + ::Gitlab::Database::Dictionary.entries.each do |entry| + allowed_schemas = entry.allow_cross_to_schemas(type) + allowed_schemas.each do |schema| + # In the context of this GitlabSchemaInfo we only need the tables that have allowed this schema + next unless schema == name + + result[entry.gitlab_schema.to_sym] ||= [] + result[entry.gitlab_schema.to_sym] << entry.key_name + end + end + result + end + end end end end diff --git a/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb index 3d630d21d4c..6e5c4fe8498 100644 --- a/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb +++ b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb @@ -82,6 +82,7 @@ module Gitlab def sli_query { gitlab_main: prometheus_alert_db_indicators_settings[sli_query_key][:main], + gitlab_main_cell: prometheus_alert_db_indicators_settings[sli_query_key][:main_cell], gitlab_ci: prometheus_alert_db_indicators_settings[sli_query_key][:ci] }.fetch(gitlab_schema) end @@ -90,6 +91,7 @@ module Gitlab def slo { gitlab_main: prometheus_alert_db_indicators_settings[slo_key][:main], + gitlab_main_cell: prometheus_alert_db_indicators_settings[slo_key][:main_cell], gitlab_ci: prometheus_alert_db_indicators_settings[slo_key][:ci] }.fetch(gitlab_schema) end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index 7cfafa1a6a6..f4f4e8ce22a 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -115,10 +115,13 @@ module Gitlab # type is used. # batch_column_name - option is for tables without primary key, in this # case another unique integer column can be used. Example: :user_id - def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id, type_cast_function: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) + setup_renamed_column( + __callee__, table, old_column, new_column, + type: type, batch_column_name: batch_column_name, type_cast_function: type_cast_function + ) with_lock_retries do install_bidirectional_triggers(table, old_column, new_column) @@ -167,7 +170,10 @@ module Gitlab def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) + setup_renamed_column( + __callee__, table, new_column, old_column, + type: type, batch_column_name: batch_column_name + ) with_lock_retries do install_bidirectional_triggers(table, old_column, new_column) @@ -198,7 +204,7 @@ module Gitlab private - def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name) + def setup_renamed_column(calling_operation, table, old_column, new_column, type:, batch_column_name:, type_cast_function: nil) if transaction_open? raise "#{calling_operation} can not be run inside a transaction" end @@ -220,7 +226,7 @@ module Gitlab check_trigger_permissions!(table) unless column_exists?(table, new_column) - create_column_from(table, old_column, new_column, type: type, batch_column_name: batch_column_name) + create_column_from(table, old_column, new_column, type: type, batch_column_name: batch_column_name, type_cast_function: type_cast_function) end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 39706582e3c..5599c65b84e 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -199,7 +199,7 @@ module Gitlab Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( - Gitlab::Database.gitlab_schemas_for_connection(connection), + gitlab_schema_from_context, job_class_name, table_name, column_name, job_arguments ) diff --git a/lib/gitlab/database/migrations/squasher.rb b/lib/gitlab/database/migrations/squasher.rb index 98fdf873aa5..3bec9eabbe2 100644 --- a/lib/gitlab/database/migrations/squasher.rb +++ b/lib/gitlab/database/migrations/squasher.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'set' +require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. module Gitlab module Database diff --git a/lib/gitlab/database/namespace_each_batch.rb b/lib/gitlab/database/namespace_each_batch.rb new file mode 100644 index 00000000000..ffc3e16061c --- /dev/null +++ b/lib/gitlab/database/namespace_each_batch.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class implements an iterator over the namespace hierarchy which uses a recursive + # depth-first algorithm. + # You can read more about the algorithm here: + # https://docs.gitlab.com/ee/development/database/poc_tree_iterator.html + # + # With the class, you can iterate over the whole hierarchy including subgroups and project namespaces + # or just iterate over the subgroups. + # + # Usage: + # + # # To invoke the iterator, you can take any group id. + # # Build the cursor object that will be used for tracking our position in the tree hierarchy. + # cursor = { current_id: 9970, depth: [9970] } + # + # # Instantiate the object. + # iterator = Gitlab::Database::NamespaceEachBatch.new(namespace_class: Namespace, cursor: cursor) + # + # iterator.each_batch(of: 100) do |ids| + # # return namespace ids which can be Group id or Namespaces::ProjectNamespace id + # puts ids + # end + # + # # When you need to break out of the iteration and continue later, you can yield the cursor as a second parameter: + # iterator.each_batch(of: 100) do |ids, new_cursor| + # save_cursor(new_cursor) && break if limit_reached? + # puts ids + # end + # + # You can build a new iterator later and resume the processing. + # + # # Building an iterator that only returns groups: + # iterator = Gitlab::Database::NamespaceEachBatch.new(namespace_class: Group, cursor: cursor) + # + class NamespaceEachBatch + PROJECTIONS = %w[current_id depth ids count index].freeze + + def initialize(namespace_class:, cursor:) + @namespace_class = namespace_class + set_cursor!(cursor) + end + + def each_batch(of: 500) + current_cursor = cursor.dup + + first_iteration = true + loop do + new_cursor, ids = load_batch(cursor: current_cursor, of: of, first_iteration: first_iteration) + break if new_cursor.nil? + + first_iteration = false + current_cursor = new_cursor + + yield ids, new_cursor + + break if new_cursor[:depth].empty? + end + end + + private + + attr_reader :namespace_class, :cursor, :namespace_id + + def load_batch(cursor:, of:, first_iteration: false) + recursive_scope = build_recursive_query(cursor, of, first_iteration) + + row = Namespace + .select(*PROJECTIONS) + .from(recursive_scope.arel.as(Namespace.table_name)).order(count: :desc) + .limit(1) + .first + + return [] unless row + + [{ current_id: row[:current_id], depth: row[:depth] }, row[:ids]] + end + + # rubocop: disable Style/AsciiComments -- Rendering a graph + # The depth-first algorithm is implemented here. Consider the following group hierarchy: + # + # ┌──┐ + # │10│ + # ┌────┴──┴────┐ + # │ │ + # ┌─┴┐ ┌┴─┐ + # │41│ │72│ + # └─┬┘ └──┘ + # │ + # ┌─┴┐ + # ┌────┤32├─────┐ + # │ └─┬┘ │ + # │ │ │ + # ┌─┴┐ ┌─┴┐ ┌┴─┐ + # │11│ │12│ │18│ + # └──┘ └──┘ └──┘ + # + # 1. Start with node 10 and look up the left-hand child nodes until reaching the leaf. (walk_down) + # 2. While walking down, record the depth in an array and also store them in the ids array. + # 3. depth: 10, 41, 32, 11 | ids: 10, 41, 32, 11 + # 4. Start collecting the ids by looking at the nodes on the deepest level. (next_elements) + # 5. This gives us the rest of the nodes on the same level (parent_id = 32 AND id > 11) + # 6. depth: 10, 41, 32, 11 | ids: 10, 41, 32, 11, 12, 18 + # 7. When done, move one level up and pop the last value from the depth. (up_one_level) + # 8. depth: 10, 41, 32 | ids: 10, 41, 32, 11, 12, 18 + # 9. Do the same, look at the nodes on the same level: no records, 32 was already collected + # 10. depth: 10, 41, 32 | ids: 10, 41, 32, 11, 12, 18 + # 11. Move one level up again and look at the nodes on the same level. + # 12. depth: 10, 41 | ids: 10, 41, 32, 11, 12, 18, 72 + # 13. Move one level up again, we reached the root node, iteration is done. + # 14. depth: 10 | ids: 10, 41, 32, 11, 12, 18, 72 + # + # By tracking the currently accessed node and the depth we can stop and restore the processing of + # the hierarchy at any point. + # + # rubocop: enable Style/AsciiComments + def build_recursive_query(cursor, of, first_iteration) + ids = first_iteration ? cursor[:current_id] : '' + + recursive_cte = Gitlab::SQL::RecursiveCTE.new(:result, + union_args: { + remove_order: false, + remove_duplicates: false + }) + + recursive_cte << base_namespace_class.select( + Arel.sql(cursor[:current_id].to_s).as('current_id'), + Arel.sql("ARRAY[#{cursor[:depth].join(',')}]::int[]").as('depth'), + Arel.sql("ARRAY[#{ids}]::int[]").as('ids'), + Arel.sql('1::bigint AS count'), + Arel.sql('0::bigint AS index') + ).from('(VALUES (1)) AS initializer_row') + .where_exists(namespace_exists_query) + + cte = Gitlab::SQL::CTE.new(:cte, base_namespace_class.select('result.*').from('result')) + + union_query = base_namespace_class.with(cte.to_arel).from_union( + walk_down, + next_elements, + up_one_level, + remove_duplicates: false, + remove_order: false + ).select(*PROJECTIONS).order(base_namespace_class.arel_table[:index].asc).limit(1) + + recursive_cte << union_query + + base_namespace_class.with + .recursive(recursive_cte.to_arel) + .from(recursive_cte.alias_to(namespace_class.arel_table)) + .select(*PROJECTIONS) + .limit(of + 1) + end + + def namespace_exists_query + Namespace.where(id: cursor[:current_id]) + end + + def walk_down + lateral_query = namespace_class + .select(:id) + .where('parent_id = cte.current_id') + .order(:id) + .limit(1) + + base_namespace_class.select( + base_namespace_class.arel_table[:id].as('current_id'), + Arel.sql("cte.depth || #{base_namespace_table}.id").as('depth'), + Arel.sql("cte.ids || #{base_namespace_table}.id").as('ids'), + Arel.sql('cte.count + 1').as('count'), + Arel.sql('1::bigint AS index') + ).from("cte, LATERAL (#{lateral_query.to_sql}) #{base_namespace_table}") + end + + def next_elements + lateral_query = namespace_class + .select(:id) + .where("#{base_namespace_table}.parent_id = cte.depth[array_length(cte.depth, 1) - 1]") + .where("#{base_namespace_table}.id > cte.depth[array_length(cte.depth, 1)]") + .order(:id) + .limit(1) + + base_namespace_class.select( + base_namespace_class.arel_table[:id].as('current_id'), + Arel.sql("cte.depth[:array_length(cte.depth, 1) - 1] || #{base_namespace_table}.id").as('depth'), + Arel.sql("cte.ids || #{base_namespace_table}.id").as('ids'), + Arel.sql('cte.count + 1').as('count'), + Arel.sql('2::bigint AS index') + ).from("cte, LATERAL (#{lateral_query.to_sql}) #{base_namespace_table}") + end + + def up_one_level + Namespace.select( + Arel.sql('cte.current_id').as('current_id'), + Arel.sql('cte.depth[:array_length(cte.depth, 1) - 1]').as('depth'), + Arel.sql('cte.ids').as('ids'), + Arel.sql('cte.count + 1').as('count'), + Arel.sql('3::bigint AS index') + ).from('cte') + .where('cte.depth <> ARRAY[]::int[]') + .limit(1) + end + + def base_namespace_class + Namespace + end + + def base_namespace_table + Namespace.quoted_table_name + end + + def set_cursor!(original_cursor) + raise ArgumentError unless original_cursor[:depth].is_a?(Array) + + @cursor = { + current_id: Integer(original_cursor[:current_id]), + depth: original_cursor[:depth].map { |value| Integer(value) } + } + end + end + end +end diff --git a/lib/gitlab/database/partitioning/int_range_partition.rb b/lib/gitlab/database/partitioning/int_range_partition.rb new file mode 100644 index 00000000000..026738a419b --- /dev/null +++ b/lib/gitlab/database/partitioning/int_range_partition.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class IntRangePartition + include Comparable + + def self.from_sql(table, partition_name, definition) + matches = definition.match(/FOR VALUES FROM \('?(?<from>\d+)'?\) TO \('?(?<to>\d+)'?\)/) + + raise ArgumentError, "Unknown partition definition: #{definition}" unless matches + + to = matches[:to].to_i + from = matches[:from].to_i + + new(table, from, to, partition_name: partition_name) + end + + attr_reader :table, :from, :to + + def initialize(table, from, to, partition_name: nil) + @table = table.to_s + @from = from + @to = to + @partition_name = partition_name + + validate! + end + + def partition_name + @partition_name || "#{table}_#{from}" + end + + def to_sql + from_sql = conn.quote(from) + to_sql = conn.quote(to) + + <<~SQL + CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} + PARTITION OF #{conn.quote_table_name(table)} + FOR VALUES FROM (#{from_sql}) TO (#{to_sql}) + SQL + end + + def ==(other) + table == other.table && partition_name == other.partition_name && from == other.from && to == other.to + end + alias_method :eql?, :== + + def hash + [table, partition_name, from, to].hash + end + + def <=>(other) + return if table != other.table + + [from.to_i, to.to_i] <=> [other.from.to_i, other.to.to_i] + end + + def holds_data? + conn.execute("SELECT 1 FROM #{fully_qualified_partition} LIMIT 1").ntuples > 0 + end + + private + + def validate! + raise '`to` statement must be greater than 0' unless to.to_i > 0 + raise '`from` statement must be greater than 0' unless from.to_i > 0 + raise '`to` must be greater than `from`' unless to.to_i > from.to_i + end + + def fully_qualified_partition + format("%s.%s", conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), + conn.quote_table_name(partition_name)) + end + + def conn + @conn ||= Gitlab::Database::SharedModel.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/int_range_strategy.rb b/lib/gitlab/database/partitioning/int_range_strategy.rb new file mode 100644 index 00000000000..605a8daa2bf --- /dev/null +++ b/lib/gitlab/database/partitioning/int_range_strategy.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class IntRangeStrategy + attr_reader :model, :partitioning_key, :partition_size + + # We create this many partitions in the future + HEADROOM = 6 + MIN_ID = 1 + + delegate :table_name, to: :model + + def initialize(model, partitioning_key, partition_size:) + @model = model + @partitioning_key = partitioning_key + @partition_size = partition_size + end + + def current_partitions + int_range_partitions = Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + IntRangePartition.from_sql(table_name, partition.name, partition.condition) + end + + int_range_partitions.sort + end + + # Check the currently existing partitions and determine which ones are missing + def missing_partitions + desired_partitions - current_partitions + end + + def extra_partitions + [] + end + + def after_adding_partitions + # No-op, required by the partition manager + end + + def validate_and_fix + # No-op, required by the partition manager + end + + private + + def are_last_partitions_empty?(number_of_partitions) + partitions = current_partitions.last(number_of_partitions) + + partitions.none?(&:holds_data?) + end + + def desired_partitions + end_id = are_partitions_syncronized? ? max_id : max_id + (HEADROOM * partition_size) # Adds 6 new partitions + + create_int_range_partitions(MIN_ID, end_id) + end + + def create_int_range_partitions(start_id, end_id) + partitions = [] + + while start_id < end_id + partitions << partition_for(lower_bound: start_id, upper_bound: start_id + partition_size, + partition_name: partition_name(start_id)) + + start_id += partition_size + end + + partitions + end + + def max_id + last_partition&.to || MIN_ID + end + + def are_partitions_syncronized? + last_partition && current_partitions.size >= HEADROOM && are_last_partitions_empty?(HEADROOM) + end + + def partition_name(lower_bound) + "#{table_name}_#{lower_bound}" + end + + def last_partition + current_partitions.last + end + + def partition_for(upper_bound:, lower_bound:, partition_name:) + IntRangePartition.new(table_name, lower_bound, upper_bound, partition_name: partition_name) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/list/convert_table.rb b/lib/gitlab/database/partitioning/list/convert_table.rb index 9889d01be76..542a7d0a78d 100644 --- a/lib/gitlab/database/partitioning/list/convert_table.rb +++ b/lib/gitlab/database/partitioning/list/convert_table.rb @@ -22,7 +22,7 @@ module Gitlab @table_name = table_name @parent_table_name = parent_table_name @partitioning_column = partitioning_column - @zero_partition_value = zero_partition_value + @zero_partition_value = Array.wrap(zero_partition_value) end def prepare_for_partitioning(async: false) @@ -126,10 +126,11 @@ module Gitlab .check_constraints .including_column(partitioning_column) - check_body = "CHECK ((#{partitioning_column} = #{zero_partition_value}))" + array_prefix = "CHECK ((#{partitioning_column} = ANY " + single_prefix = "CHECK ((#{partitioning_column} = #{zero_partition_value.join(',')}))" constraints_on_column.find do |constraint| - constraint.definition.start_with?(check_body) + constraint.definition.start_with?(array_prefix, single_prefix) end end @@ -138,14 +139,14 @@ module Gitlab raise UnableToPartition, <<~MSG Table #{table_name} is not ready for partitioning. - Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value}) + Before partitioning, a check constraint must enforce that (#{partitioning_column} IN (#{zero_partition_value.join(',')})) MSG end def add_partitioning_check_constraint(async: false) return validate_partitioning_constraint_synchronously if partitioning_constraint.present? - check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}" + check_body = "#{partitioning_column} IN (#{zero_partition_value.join(',')})" # Any constraint name would work. The constraint is found based on its definition before partitioning migration_context.add_check_constraint( table_name, check_body, PARTITIONING_CONSTRAINT_NAME, @@ -214,7 +215,7 @@ module Gitlab <<~SQL ALTER TABLE #{quote_table_name(parent_table_name)} ATTACH PARTITION #{table_name} - FOR VALUES IN (#{zero_partition_value}) + FOR VALUES IN (#{zero_partition_value.join(',')}) SQL end diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb index 3196dd20356..daafda8e7be 100644 --- a/lib/gitlab/database/partitioning_migration_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers.rb @@ -6,6 +6,7 @@ module Gitlab include ForeignKeyHelpers include TableManagementHelpers include IndexHelpers + include UniquenessHelpers end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index b486ddb8e76..d906ad45430 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -368,7 +368,7 @@ module Gitlab end def make_partitioned_table_name(table) - tmp_table_name("#{table}_part") + tmp_table_name(table) end def make_archived_table_name(table) diff --git a/lib/gitlab/database/partitioning_migration_helpers/uniqueness_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/uniqueness_helpers.rb new file mode 100644 index 00000000000..1c33371057e --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/uniqueness_helpers.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + module UniquenessHelpers + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::SchemaHelpers + + def ensure_unique_id(table_name) + function_name = "assign_#{table_name}_id_value" + trigger_name = "assign_#{table_name}_id_trigger" + + return if trigger_exists?(table_name, trigger_name) + + change_column_default(table_name, :id, nil) + + create_trigger_function(function_name) do + <<~SQL + IF NEW."id" IS NOT NULL THEN + RAISE WARNING 'Manually assigning ids is not allowed, the value will be ignored'; + END IF; + NEW."id" := nextval('#{existing_sequence(table_name)}'::regclass); + RETURN NEW; + SQL + end + + create_trigger(table_name, trigger_name, function_name, fires: 'BEFORE INSERT') + end + + private + + def existing_sequence(table_name) + Gitlab::Database::PostgresSequence.by_table_name(table_name).first + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index fb25cb70e57..f16b6ca2177 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -86,7 +86,7 @@ module Gitlab end return unless self.in_transaction? - return if in_factory_bot_create? + return if Thread.current[:factory_bot_objects] && Thread.current[:factory_bot_objects] > 0 # PgQuery might fail in some cases due to limited nesting: # https://github.com/pganalyze/pg_query/issues/209 @@ -192,20 +192,6 @@ module Gitlab def self.in_transaction? context[:transaction_depth_by_db].values.any?(&:positive?) end - - # We ignore execution in the #create method from FactoryBot - # because it is not representative of real code we run in - # production. There are far too many false positives caused - # by instantiating objects in different `gitlab_schema` in a - # FactoryBot `create`. - def self.in_factory_bot_create? - Rails.env.test? && caller_locations.any? do |l| - l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || - l.path.end_with?('lib/factory_bot/strategy/create.rb') || - l.path.end_with?('lib/factory_bot/strategy/build.rb') || - l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' - end - end end end end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 000a1f50a92..6e6080a0543 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -22,7 +22,9 @@ module Gitlab notifications: 'Notifications', current_user_todos: 'Current user todos', award_emoji: 'Award emoji', - linked_items: 'Linked items' + linked_items: 'Linked items', + color: 'Color', + rolledup_dates: 'Rolledup dates' }.freeze WIDGETS_FOR_TYPE = { @@ -126,7 +128,9 @@ module Gitlab :notifications, :current_user_todos, :award_emoji, - :linked_items + :linked_items, + :color, + :rolledup_dates ], ticket: [ :assignees, diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb index 4e3b685c06c..ec0d9c3f36e 100644 --- a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb +++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb @@ -66,7 +66,10 @@ module Gitlab def self.find_or_create_type(name) type = ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) - return type if type + if type + type.clear_reactive_cache! + return type + end Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index db60128b979..d4178173e03 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -22,11 +22,19 @@ module Gitlab LINKERS.find { |linker| linker.support?(blob_name) } end - def self.link(blob_name, plain_text, highlighted_text) + def self.link(blob_name, plain_text, highlighted_text, used_on: :blob) linker = linker(blob_name) return highlighted_text unless linker + usage_counter.increment(used_on: used_on) linker.link(plain_text, highlighted_text) end + + def self.usage_counter + Gitlab::Metrics.counter( + :dependency_linker_usage, + 'The number of times dependency linker is used' + ) + end end end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 74bec55253f..2c9b559c8dc 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -31,13 +31,15 @@ module Gitlab end def external_url(name, external_ref) - return if GIT_INVALID_URL_REGEX.match?(external_ref) + ref = external_ref.to_s - case external_ref + return if GIT_INVALID_URL_REGEX.match?(ref) + + case ref when /\A#{URL_REGEX}\z/o - external_ref + ref when /\A#{REPO_REGEX}\z/o - github_url(external_ref) + github_url(ref) else package_url(name) end diff --git a/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb index 37abad81305..77461db7d7d 100644 --- a/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/paginated_merge_request_diff.rb @@ -15,8 +15,8 @@ module Gitlab delegate :limit_value, :current_page, :next_page, :prev_page, :total_count, :total_pages, to: :paginated_collection - def initialize(merge_request_diff, page, per_page) - super(merge_request_diff, diff_options: nil) + def initialize(merge_request_diff, page, per_page, diff_options) + super(merge_request_diff, diff_options: diff_options) @paginated_collection = load_paginated_collection(page, per_page) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index d2524ae1761..39da9c4e7c8 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -90,7 +90,8 @@ module Gitlab rich_line = syntax_highlighter(diff_line).highlight( diff_line.text(prefix: false), plain: plain, - context: { line_number: diff_line.line } + context: { line_number: diff_line.line }, + used_on: :diff ) # Only update text if line is found. This will prevent @@ -143,7 +144,7 @@ module Gitlab blob.load_all_data! - blob.present.highlight.lines + blob.present.highlight(used_on: :diff).lines end def blobs_too_large? diff --git a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb index ad709a79f30..b4b7d572901 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb @@ -4,7 +4,7 @@ module Gitlab module Rendered module Notebook module DiffFileHelper - require 'set' + require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. EMBEDDED_IMAGE_PATTERN = ' ![](data:image' diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index b080cb197d4..7ad4cf96dd4 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -40,7 +40,7 @@ module Gitlab "--broken encoding: #{encoding}" end - def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil) + def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN) return if data.nil? CharlockHolmes::EncodingDetector.new(limit).detect(data) @@ -54,8 +54,8 @@ module Gitlab # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), # which is what we use below to keep a consistent behavior. - def detect_libgit2_binary?(data, cache_key: nil) - detect = detect_encoding(data, limit: 8000, cache_key: cache_key) + def detect_libgit2_binary?(data) + detect = detect_encoding(data, limit: 8000) detect && detect[:type] == :binary end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 239aee97378..c66edfdda10 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -158,10 +158,12 @@ module Gitlab end def process_exception(exception, extra:, tags: {}, trackers: default_trackers) - context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra, tags) + Gitlab::Utils.allow_within_concurrent_ruby do + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra, tags) - trackers.each do |tracker| - tracker.capture_exception(exception, **context_payload) + trackers.each do |tracker| + tracker.capture_exception(exception, **context_payload) + end end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index cc8cfd827f1..a0b6318e066 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'set' +require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. module Gitlab module ErrorTracking diff --git a/lib/gitlab/event_store/event.rb b/lib/gitlab/event_store/event.rb index ba82ae6dd6a..aecb4fd17c3 100644 --- a/lib/gitlab/event_store/event.rb +++ b/lib/gitlab/event_store/event.rb @@ -47,7 +47,7 @@ module Gitlab def validate_schema! if self.class.json_schema_valid.nil? - self.class.json_schema_valid = JSONSchemer.schema(self.class.json_schema).valid?(schema) + self.class.json_schema_valid = JSONSchemer.schema(Event.json_schema).valid?(schema) end return if self.class.json_schema_valid == true @@ -60,8 +60,12 @@ module Gitlab raise Gitlab::EventStore::InvalidEvent, "Event data must be a Hash" end - unless JSONSchemer.schema(schema).valid?(data.deep_stringify_keys) - raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{schema}" + errors = JSONSchemer.schema(schema).validate(data.deep_stringify_keys).map do |error| + JSONSchemer::Errors.pretty(error) + end + + unless errors.empty? + raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{errors.inspect}" end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index b586c4b5892..0a56cde8cad 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'set' +require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. module Gitlab # Module that can be used to detect if a path points to a special file such as diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8cbd1a4ce72..894811ecd3c 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -7,7 +7,8 @@ module Gitlab # The ID of empty tree. # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' - BLANK_SHA = ('0' * 40).freeze + SHA1_BLANK_SHA = ('0' * 40).freeze + SHA256_BLANK_SHA = ('0' * 64).freeze COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/ TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" @@ -79,7 +80,7 @@ module Gitlab end def blank_ref?(ref) - ref == BLANK_SHA + ref == SHA1_BLANK_SHA end def commit_id?(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index aa59caa4268..6bbb0f4b7a0 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -110,8 +110,8 @@ module Gitlab end end - def binary?(data, cache_key: nil) - EncodingHelper.detect_libgit2_binary?(data, cache_key: cache_key) + def binary?(data) + EncodingHelper.detect_libgit2_binary?(data) end def size_could_be_lfs?(size) diff --git a/lib/gitlab/git/changed_path.rb b/lib/gitlab/git/changed_path.rb index 033779466f6..11fc7f5bb66 100644 --- a/lib/gitlab/git/changed_path.rb +++ b/lib/gitlab/git/changed_path.rb @@ -3,16 +3,28 @@ module Gitlab module Git class ChangedPath - attr_reader :status, :path + attr_reader :status, :path, :old_mode, :new_mode - def initialize(status:, path:) + def initialize(status:, path:, old_mode:, new_mode:) @status = status @path = path + @old_mode = old_mode + @new_mode = new_mode end def new_file? status == :ADDED end + + def submodule_change? + # The file mode 160000 represents a "Gitlink" or a git submodule. + # The first two digits can be used to distinguish it from regular files. + # + # 160000 -> 16 -> gitlink + # 100644 -> 10 -> regular file + + [old_mode, new_mode].any? { |mode| mode.starts_with?('16') } + end end end end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index c6d678c9432..1c60345aea9 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -48,9 +48,8 @@ module Gitlab changed_paths = @repository .find_changed_paths([Gitlab::Git::DiffTree.new(@base.id, @head.id)]) - .map(&:path) - @repository.detect_generated_files(@base.id, changed_paths) + @repository.detect_generated_files(@base.id, @head.id, changed_paths) end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index e753d356bc6..a8e680e539b 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -299,9 +299,9 @@ module Gitlab @old_path = encode!(gitaly_diff.from_path.dup) @a_mode = gitaly_diff.old_mode.to_s(8) @b_mode = gitaly_diff.new_mode.to_s(8) - @new_file = gitaly_diff.from_id == BLANK_SHA + @new_file = gitaly_diff.from_id == SHA1_BLANK_SHA @renamed_file = gitaly_diff.from_path != gitaly_diff.to_path - @deleted_file = gitaly_diff.to_id == BLANK_SHA + @deleted_file = gitaly_diff.to_id == SHA1_BLANK_SHA @too_large = gitaly_diff.too_large if gitaly_diff.respond_to?(:too_large) gitaly_overflow = gitaly_diff.try(:overflow_marker) @overflow = Diff.collect_patch_overage? && gitaly_overflow diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb index 3d533a5185f..fc10b93991c 100644 --- a/lib/gitlab/git/push.rb +++ b/lib/gitlab/git/push.rb @@ -9,8 +9,8 @@ module Gitlab def initialize(project, oldrev, newrev, ref) @project = project - @oldrev = oldrev.presence || Gitlab::Git::BLANK_SHA - @newrev = newrev.presence || Gitlab::Git::BLANK_SHA + @oldrev = oldrev.presence || Gitlab::Git::SHA1_BLANK_SHA + @newrev = newrev.presence || Gitlab::Git::SHA1_BLANK_SHA @ref = ref end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 312e05b5f54..1bf796e167d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -398,7 +398,7 @@ module Gitlab end def new_blobs(newrevs, dynamic_timeout: nil) - newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } + newrevs = Array.wrap(newrevs).reject { |rev| rev.blank? || rev == ::Gitlab::Git::SHA1_BLANK_SHA } return [] if newrevs.empty? newrevs = newrevs.uniq.sort @@ -416,7 +416,7 @@ module Gitlab # GitalyClient.medium_timeout and dynamic timeout if the dynamic # timeout is set, otherwise it'll always use the medium timeout. def blobs(revisions, with_paths: false, dynamic_timeout: nil) - revisions = revisions.reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } + revisions = revisions.reject { |rev| rev.blank? || rev == ::Gitlab::Git::SHA1_BLANK_SHA } return [] if revisions.blank? @@ -458,7 +458,7 @@ module Gitlab @raw_changes_between[[old_rev, new_rev]] ||= begin - return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA + return [] if new_rev.blank? || new_rev == Gitlab::Git::SHA1_BLANK_SHA wrapped_gitaly_errors do gitaly_repository_client.raw_changes_between(old_rev, new_rev) @@ -752,7 +752,7 @@ module Gitlab # new_sha # reference: # - # When new_sha is Gitlab::Git::BLANK_SHA, then this will be deleted + # When new_sha is Gitlab::Git::SHA1_BLANK_SHA, then this will be deleted def update_refs(ref_list) wrapped_gitaly_errors do gitaly_ref_client.update_refs(ref_list: ref_list) if ref_list.any? @@ -1089,6 +1089,10 @@ module Gitlab @praefect_info_client ||= Gitlab::GitalyClient::PraefectInfoService.new(self) end + def gitaly_analysis_client + @gitaly_analysis_client ||= Gitlab::GitalyClient::AnalysisService.new(self) + end + def branch_names_contains_sha(sha, limit: 0) gitaly_ref_client.branch_names_contains_sha(sha, limit: limit) end @@ -1202,6 +1206,12 @@ module Gitlab end end + def object_format + wrapped_gitaly_errors do + gitaly_repository_client.object_format.format + end + end + def get_file_attributes(revision, file_paths, attributes) wrapped_gitaly_errors do gitaly_repository_client @@ -1211,24 +1221,49 @@ module Gitlab end end - def object_format - wrapped_gitaly_errors do - gitaly_repository_client.object_format.format - end - end - # rubocop: disable CodeReuse/ActiveRecord -- not an active record operation - def detect_generated_files(revision, paths) - return Set.new if paths.blank? - - get_file_attributes(revision, paths, Gitlab::Git::ATTRIBUTE_OVERRIDES[:generated]) + def detect_generated_files(base, head, changed_paths) + return Set.new if changed_paths.blank? + + # Check .gitattributes overrides first + checked_files = get_file_attributes( + base, + changed_paths.map(&:path), + Gitlab::Git::ATTRIBUTE_OVERRIDES[:generated] + ).map { |attrs| { path: attrs[:path], generated: attrs[:value] == "set" } } + + # Check automatic generated file detection for the remaining paths + overridden_paths = checked_files.pluck(:path) + remainder = changed_paths.reject { |changed_path| overridden_paths.include?(changed_path.path) } + checked_files += check_blobs_generated(base, head, remainder) if remainder.present? + + checked_files + .select { |attrs| attrs[:generated] } .pluck(:path) .to_set + + rescue Gitlab::Git::CommandError => e + # An exception can be raised due to an unknown revision or paths. + Gitlab::ErrorTracking.track_exception( + e, + gl_project_path: @gl_project_path, + base: base, + head: head, + paths: changed_paths.map(&:path) + ) + + Set.new end # rubocop: enable CodeReuse/ActiveRecord private + def check_blobs_generated(base, head, changed_paths) + wrapped_gitaly_errors do + gitaly_analysis_client.check_blobs_generated(base, head, changed_paths) + end + end + def repository_info_size_megabytes bytes = gitaly_repository_client.repository_info.size diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 4747ab55c63..6365ac941de 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -12,15 +12,17 @@ module Gitlab class << self # Get list of tree objects # for repository based on commit sha and path - def where( - repository, sha, path = nil, recursive = false, skip_flat_paths = true, rescue_not_found = true, - pagination_params = nil) + def tree_entries( + repository:, + sha:, + path: nil, + recursive: false, + skip_flat_paths: true, + rescue_not_found: true, + pagination_params: nil + ) path = nil if path == '' || path == '/' - tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params) - end - - def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil) wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries( repository, sha, path, recursive, skip_flat_paths, pagination_params) diff --git a/lib/gitlab/gitaly_client/analysis_service.rb b/lib/gitlab/gitaly_client/analysis_service.rb new file mode 100644 index 00000000000..9e8e6474a20 --- /dev/null +++ b/lib/gitlab/gitaly_client/analysis_service.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class AnalysisService + include Gitlab::EncodingHelper + include WithFeatureFlagActors + + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + + self.repository_actor = repository + end + + def check_blobs_generated(base, head, changed_paths) + request_enum = Enumerator.new do |y| + changed_paths.each_slice(100).with_index do |paths_subset, i| + blobs = paths_subset.filter_map do |changed_path| + # Submodule changes should be ignored as the blob won't exist + next if changed_path.submodule_change? + + # The Blob won't exist in the base if the file is newly added. + # We can use the head to get the blob to handle both added or deleted files. + prefix = changed_path.new_file? ? head : base + revision = "#{prefix}:#{changed_path.path}" + + Gitaly::CheckBlobsGeneratedRequest::Blob.new( + revision: encode_binary(revision), + path: encode_binary(changed_path.path) + ) + end + + next if blobs.blank? + + params = { blobs: blobs } + # Repository is only needed for the first request + params[:repository] = @gitaly_repo if i == 0 + + y.yield Gitaly::CheckBlobsGeneratedRequest.new(**params) + end + end + + return [] if request_enum.count == 0 + + response = gitaly_client_call( + @repository.storage, + :analysis_service, + :check_blobs_generated, + request_enum, + timeout: GitalyClient.medium_timeout + ) + + result = [] + response.each do |msg| + msg.blobs.each do |blob| + path = blob.revision.split(":", 2).last + result << { path: path, generated: blob.generated } + end + end + + result + end + end + end +end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index 6c51b4cf8c6..95053207d1a 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -41,7 +41,7 @@ module Gitlab size: blob_data[:size], commit_id: blob_data[:revision], data: data, - binary: Gitlab::Git::Blob.binary?(data, cache_key: blob_data[:oid]) + binary: Gitlab::Git::Blob.binary?(data) ) end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 3949e8e6416..658801cdedb 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -284,7 +284,9 @@ module Gitlab msg.paths.map do |path| Gitlab::Git::ChangedPath.new( status: path.status, - path: EncodingHelper.encode!(path.path) + path: EncodingHelper.encode!(path.path), + old_mode: path.old_mode.to_s(8), + new_mode: path.new_mode.to_s(8) ) end end diff --git a/lib/gitlab/github_gists_import/importer/gists_importer.rb b/lib/gitlab/github_gists_import/importer/gists_importer.rb index 08744dbaf5f..4ed6b2db5f8 100644 --- a/lib/gitlab/github_gists_import/importer/gists_importer.rb +++ b/lib/gitlab/github_gists_import/importer/gists_importer.rb @@ -39,7 +39,7 @@ module Gitlab end def fetch_gists_to_import - page_counter = Gitlab::GithubImport::PageCounter.new(user, :gists, 'github-gists-importer') + page_counter = Gitlab::Import::PageCounter.new(user, :gists, 'github-gists-importer') collection = [] client.each_page(:gists, nil, page: page_counter.current) do |page| diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb index df9c6c8342d..e9192b97506 100644 --- a/lib/gitlab/github_import/attachments_downloader.rb +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -11,7 +11,7 @@ module Gitlab UnsupportedAttachmentError = Class.new(StandardError) FILENAME_SIZE_LIMIT = 255 # chars before the extension - DEFAULT_FILE_SIZE_LIMIT = 25.megabytes + DEFAULT_FILE_SIZE_LIMIT = Gitlab::CurrentSettings.max_attachment_size.megabytes TMP_DIR = File.join(Dir.tmpdir, 'github_attachments').freeze attr_reader :file_url, :filename, :file_size_limit, :options @@ -26,7 +26,6 @@ module Gitlab end def perform - validate_content_length validate_filepath download_url = get_assets_download_redirection_url @@ -46,11 +45,6 @@ module Gitlab raise DownloadError, message end - def response_headers - @response_headers ||= - Gitlab::HTTP.perform_request(Net::HTTP::Head, file_url, {}).headers - end - # Github /assets redirection link will redirect to aws which has its own authorization. # Keeping our bearer token will cause request rejection # eg. Only one auth mechanism allowed; only the X-Amz-Algorithm query parameter, @@ -78,7 +72,19 @@ module Gitlab def download_from(url) file = File.open(filepath, 'wb') - Gitlab::HTTP.perform_request(Net::HTTP::Get, url, stream_body: true) { |batch| file.write(batch) } + + Gitlab::HTTP.perform_request(Net::HTTP::Get, url, stream_body: true) do |chunk| + next if [301, 302, 303, 307, 308].include?(chunk.code) + + raise DownloadError, "Error downloading file from #{url}. Error code: #{chunk.code}" if chunk.code != 200 + + file.write(chunk) + validate_size!(file.size) + rescue Gitlab::GithubImport::AttachmentsDownloader::DownloadError + delete + raise + end + file end diff --git a/lib/gitlab/github_import/events_cache.rb b/lib/gitlab/github_import/events_cache.rb new file mode 100644 index 00000000000..0986ccfaed1 --- /dev/null +++ b/lib/gitlab/github_import/events_cache.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class EventsCache + MAX_NUMBER_OF_EVENTS = 100 + MAX_EVENT_SIZE = 100.kilobytes + + def initialize(project) + @project = project + end + + # Add issue event as JSON to the cache + # + # @param record [ActiveRecord::Model] Model that responds to :iid + # @param event [GitLab::GitHubImport::Representation::IssueEvent] + def add(record, issue_event) + json = issue_event.to_hash.to_json + + if json.bytesize > MAX_EVENT_SIZE + Logger.warn( + message: 'Event too large to cache', + project_id: project.id, + github_identifiers: issue_event.github_identifiers + ) + + return + end + + Gitlab::Cache::Import::Caching.list_add(events_cache_key(record), json, limit: MAX_NUMBER_OF_EVENTS) + end + + # Reads issue events from cache + # + # @param record [ActiveRecord::Model] Model that responds to :iid + # @retun [Array<GitLab::GitHubImport::Representation::IssueEvent>] List of issue events + def events(record) + events = Gitlab::Cache::Import::Caching.values_from_list(events_cache_key(record)).map do |event| + Representation::IssueEvent.from_json_hash(Gitlab::Json.parse(event)) + end + + events.sort_by(&:created_at) + end + + # Deletes the cache + # + # @param record [ActiveRecord::Model] Model that responds to :iid + def delete(record) + Gitlab::Cache::Import::Caching.del(events_cache_key(record)) + end + + private + + attr_reader :project + + def events_cache_key(record) + "github-importer/events/#{project.id}/#{record.class.name}/#{record.iid}" + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/base_importer.rb b/lib/gitlab/github_import/importer/attachments/base_importer.rb index eaff99aed43..844008f8087 100644 --- a/lib/gitlab/github_import/importer/attachments/base_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/base_importer.rb @@ -16,9 +16,11 @@ module Gitlab batch.each do |record| next if already_imported?(record) - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + if has_attachments?(record) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - yield record + yield record + end # We mark the object as imported immediately so we don't end up # scheduling it multiple times. @@ -48,6 +50,12 @@ module Gitlab def object_representation(object) representation_class.from_db_record(object) end + + def has_attachments?(object) + return true if Feature.disabled?(:github_importer_attachments, project, type: :gitlab_com_derisk) + + object_representation(object).has_attachments? + end end end end diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb index 8218acf2bfb..1ebafec5afc 100644 --- a/lib/gitlab/github_import/importer/events/base_importer.rb +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -10,6 +10,7 @@ module Gitlab # client - An instance of `Gitlab::GithubImport::Client`. def initialize(project, client) @project = project + @client = client @user_finder = UserFinder.new(project, client) end @@ -20,7 +21,7 @@ module Gitlab private - attr_reader :project, :user_finder + attr_reader :project, :user_finder, :client def author_id(issue_event, author_key: :actor) user_finder.author_id_for(issue_event, author_key: author_key).first @@ -42,6 +43,10 @@ module Gitlab belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id { belongs_to_key => issuable_db_id(issue_event) } end + + def import_settings + @import_settings ||= Gitlab::GithubImport::Settings.new(project) + end end end end diff --git a/lib/gitlab/github_import/importer/events/commented.rb b/lib/gitlab/github_import/importer/events/commented.rb new file mode 100644 index 00000000000..c9ebc31fa06 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/commented.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Commented < BaseImporter + def execute(issue_event) + return true unless import_settings.extended_events? + + note = Representation::Note.from_json_hash( + noteable_id: issue_event.issuable_id, + noteable_type: issue_event.issuable_type, + author: issue_event.actor&.to_hash, + note: issue_event.body, + created_at: issue_event.created_at, + updated_at: issue_event.updated_at, + note_id: issue_event.id + ) + + NoteImporter.new(note, project, client).execute + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/merged.rb b/lib/gitlab/github_import/importer/events/merged.rb index 6189fa8f429..702ea7f1fd5 100644 --- a/lib/gitlab/github_import/importer/events/merged.rb +++ b/lib/gitlab/github_import/importer/events/merged.rb @@ -6,6 +6,8 @@ module Gitlab module Events class Merged < BaseImporter def execute(issue_event) + create_note(issue_event) if import_settings.extended_events? + create_event(issue_event) create_state_event(issue_event) end @@ -37,6 +39,17 @@ module Gitlab ResourceStateEvent.create!(attrs) end + + def create_note(issue_event) + pull_request = Representation::PullRequest.from_json_hash({ + merged_by: issue_event.actor&.to_hash, + merged_at: issue_event.created_at, + iid: issue_event.issuable_id, + state: :closed + }) + + PullRequests::MergedByImporter.new(pull_request, project, client).execute + end end end end diff --git a/lib/gitlab/github_import/importer/events/reviewed.rb b/lib/gitlab/github_import/importer/events/reviewed.rb new file mode 100644 index 00000000000..1c0e8a9e6e8 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/reviewed.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Reviewed < BaseImporter + def execute(issue_event) + return true unless import_settings.extended_events? + + review = Representation::PullRequestReview.new( + merge_request_iid: issue_event.issuable_id, + author: issue_event.actor&.to_hash, + note: issue_event.body.to_s, + review_type: issue_event.state.upcase, # On timeline API, the state is in lower case + submitted_at: issue_event.submitted_at, + review_id: issue_event.id + ) + + PullRequests::ReviewImporter.new(review, project, client).execute({ add_reviewer: false }) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index d20482eca6f..9f15e9a25d8 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -22,6 +22,17 @@ module Gitlab unlabeled ].freeze + EXTENDED_SUPPORTED_EVENTS = SUPPORTED_EVENTS + %w[ + commented + reviewed + ].freeze + + EVENT_COUNTER_MAP = { + 'commented' => 'note', + 'reviewed' => 'pull_request_review', + 'merged' => 'pull_request_merged_by' + }.freeze + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. @@ -65,6 +76,10 @@ module Gitlab Gitlab::GithubImport::Importer::Events::ChangedReviewer when 'merged' Gitlab::GithubImport::Importer::Events::Merged + when 'commented' + Gitlab::GithubImport::Importer::Events::Commented + when 'reviewed' + Gitlab::GithubImport::Importer::Events::Reviewed end end end diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb index 26472b0d468..36a256bbef5 100644 --- a/lib/gitlab/github_import/importer/note_attachments_importer.rb +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -16,10 +16,9 @@ module Gitlab end def execute - attachments = MarkdownText.fetch_attachments(note_text.text) - return if attachments.blank? + return unless note_text.has_attachments? - new_text = attachments.reduce(note_text.text) do |text, attachment| + new_text = note_text.attachments.reduce(note_text.text) do |text, attachment| new_url = gitlab_attachment_link(attachment) text.gsub(attachment.url, new_url) end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb index 6df130eb6e8..384880651ef 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb @@ -14,10 +14,12 @@ module Gitlab @review = review @project = project @client = client - @merge_request = project.merge_requests.find_by_id(review.merge_request_id) + @merge_request = project.merge_requests.find_by_iid(review.merge_request_iid) end - def execute + def execute(options = {}) + options = { add_reviewer: true }.merge(options) + user_finder = GithubImport::UserFinder.new(project, client) gitlab_user_id = user_finder.user_id_for(review.author) @@ -25,7 +27,7 @@ module Gitlab if gitlab_user_id add_review_note!(gitlab_user_id) add_approval!(gitlab_user_id) - add_reviewer!(gitlab_user_id) + add_reviewer!(gitlab_user_id) if options[:add_reviewer] else add_complementary_review_note!(project.creator_id) end diff --git a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb index 347423b0e21..62c9e6469d7 100644 --- a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb @@ -72,7 +72,7 @@ module Gitlab merge_requests_to_import.find_each do |merge_request| # The page counter needs to be scoped by merge request to avoid skipping # pages of reviews from already imported merge requests. - page_counter = PageCounter.new(project, page_counter_id(merge_request)) + page_counter = Gitlab::Import::PageCounter.new(project, page_counter_id(merge_request)) repo = project.import_source options = collection_options.merge(page: page_counter.current) diff --git a/lib/gitlab/github_import/importer/replay_events_importer.rb b/lib/gitlab/github_import/importer/replay_events_importer.rb new file mode 100644 index 00000000000..83578cf7672 --- /dev/null +++ b/lib/gitlab/github_import/importer/replay_events_importer.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReplayEventsImporter + SUPPORTED_EVENTS = %w[review_request_removed review_requested].freeze + + # replay_event - An instance of `Gitlab::GithubImport::Representation::ReplayEvent`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(replay_event, project, client) + @project = project + @client = client + @replay_event = replay_event + end + + def execute + association = case replay_event.issuable_type + when 'MergeRequest' + project.merge_requests.find_by_iid(replay_event.issuable_iid) + end + + return unless association + + events_cache = EventsCache.new(project) + + handle_review_requests(association, events_cache.events(association)) + + events_cache.delete(association) + end + + private + + attr_reader :project, :client, :replay_event + + def handle_review_requests(association, events) + reviewers = {} + + events.each do |event| + case event.event + when 'review_requested' + reviewers[event.requested_reviewer.login] = event.requested_reviewer.to_hash if event.requested_reviewer + when 'review_request_removed' + reviewers[event.requested_reviewer.login] = nil if event.requested_reviewer + end + end + + representation = Representation::PullRequests::ReviewRequests.from_json_hash( + merge_request_id: association.id, + merge_request_iid: association.iid, + users: reviewers.values.compact + ) + + Importer::PullRequests::ReviewRequestImporter.new(representation, project, client).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index d7fa098a775..126a0b8fa4a 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -30,9 +30,11 @@ module Gitlab compose_associated_id!(parent_record, associated) - return if already_imported?(associated) || importer_class::SUPPORTED_EVENTS.exclude?(associated[:event]) + return if already_imported?(associated) || supported_events.exclude?(associated[:event]) - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + cache_event(parent_record, associated) + + increment_object_counter(associated[:event]) pull_request = parent_record.is_a? MergeRequest associated[:issue] = { number: parent_record.iid, pull_request: pull_request } @@ -64,6 +66,12 @@ module Gitlab :issue_event end + def increment_object_counter(event_name) + counter_type = importer_class::EVENT_COUNTER_MAP[event_name] if import_settings.extended_events? + counter_type ||= object_type + Gitlab::GithubImport::ObjectCounter.increment(project, counter_type, :fetched) + end + def collection_method :issue_timeline end @@ -98,6 +106,43 @@ module Gitlab event[:id] = "cross-reference##{issuable.iid}-in-#{event.dig(:source, :issue, :id)}" end + + def import_settings + @import_settings ||= Gitlab::GithubImport::Settings.new(project) + end + + def after_batch_processed(parent) + return unless import_settings.extended_events? + + events = events_cache.events(parent) + + return if events.empty? + + hash = Representation::ReplayEvent.new(issuable_type: parent.class.name.to_s, issuable_iid: parent.iid) + .to_hash.deep_stringify_keys + ReplayEventsWorker.perform_async(project.id, hash, job_waiter.key.to_s) + job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) + end + + def supported_events + return importer_class::EXTENDED_SUPPORTED_EVENTS if import_settings.extended_events? + + importer_class::SUPPORTED_EVENTS + end + + def cache_event(parent_record, associated) + return unless import_settings.extended_events? + + return if Importer::ReplayEventsImporter::SUPPORTED_EVENTS.exclude?(associated[:event]) + + representation = representation_class.from_api_response(associated) + + events_cache.add(parent_record, representation) + end + + def events_cache + @events_cache ||= EventsCache.new(project) + end end end end diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 50cad1aae19..a456e198afd 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -15,9 +15,9 @@ module Gitlab private def calculate_job_delay(job_index) - multiplier = (job_index / parallel_import_batch[:size]) + multiplier = (job_index / parallel_import_batch[:size].to_f) - (multiplier * parallel_import_batch[:delay]).to_i + 1 + (multiplier * parallel_import_batch[:delay]) + 1 end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 8e9d6d8dd50..5880aa04358 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -41,6 +41,8 @@ module Gitlab def fetch_attachments(text) attachments = [] + return attachments if text.nil? + doc = CommonMarker.render_doc(text) doc.walk do |node| diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index ce93b5203df..2286dcf767f 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -8,6 +8,8 @@ module Gitlab attr_reader :project, :client, :page_counter, :already_imported_cache_key, :job_waiter_cache_key, :job_waiter_remaining_cache_key + attr_accessor :job_started_at, :enqueued_job_counter + # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = 'github-importer/already-imported/%{project}/%{collection}' @@ -25,7 +27,7 @@ module Gitlab @project = project @client = client @parallel = parallel - @page_counter = PageCounter.new(project, collection_method) + @page_counter = Gitlab::Import::PageCounter.new(project, collection_method) @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, collection: collection_method) @job_waiter_cache_key = format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method) @@ -91,14 +93,15 @@ module Gitlab end def spread_parallel_import - enqueued_job_counter = 0 + self.job_started_at = Time.current + self.enqueued_job_counter = 0 each_object_to_import do |object| repr = object_representation(object) - job_delay = calculate_job_delay(enqueued_job_counter) sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash.deep_stringify_keys, job_waiter.key.to_s) - enqueued_job_counter += 1 + + self.enqueued_job_counter += 1 job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) end @@ -246,6 +249,14 @@ module Gitlab JobWaiter.new(jobs_remaining, key) end end + + def job_delay + runtime = Time.current - job_started_at + + delay = calculate_job_delay(enqueued_job_counter) - runtime + + delay > 0 ? delay : 1.0.second + end end end end diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 30608112f85..fc3bc5a48ef 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -8,7 +8,8 @@ module Gitlab expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, :milestone_title, :issue, :source, :assignee, :review_requester, - :requested_reviewer, :created_at + :requested_reviewer, :created_at, :updated_at, :submitted_at, + :state, :body # attributes - A Hash containing the event details. The keys of this # Hash (and any nested hashes) must be symbols. @@ -51,7 +52,11 @@ module Gitlab assignee: user_representation(event[:assignee]), requested_reviewer: user_representation(event[:requested_reviewer]), review_requester: user_representation(event[:review_requester]), - created_at: event[:created_at] + created_at: event[:created_at], + updated_at: event[:updated_at], + submitted_at: event[:submitted_at], + state: event[:state], + body: event[:body] ) end diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb index 43e18a923d6..79bef4ec363 100644 --- a/lib/gitlab/github_import/representation/note_text.rb +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -55,6 +55,14 @@ module Gitlab }.merge(record_type_specific_attribute) end + def has_attachments? + attachments.present? + end + + def attachments + @attachments ||= MarkdownText.fetch_attachments(text) + end + private def record_type_specific_attribute diff --git a/lib/gitlab/github_import/representation/replay_event.rb b/lib/gitlab/github_import/representation/replay_event.rb new file mode 100644 index 00000000000..2d71c26abbb --- /dev/null +++ b/lib/gitlab/github_import/representation/replay_event.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class ReplayEvent + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :issuable_type, :issuable_iid + + def self.from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { + issuable_type: issuable_type, + issuable_iid: issuable_iid + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 3947ae3c63d..da5833df3a1 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -38,8 +38,13 @@ module Gitlab } }.freeze - def self.stages_array - OPTIONAL_STAGES.map do |stage_name, data| + def self.stages_array(current_user) + deprecated_options = %i[single_endpoint_issue_events_import] + + OPTIONAL_STAGES.filter_map do |stage_name, data| + next if deprecated_options.include?(stage_name) && + Feature.enabled?(:github_import_extended_events, current_user) + { name: stage_name.to_s, label: s_(format("GitHubImport|%{text}", text: data[:label])), @@ -61,7 +66,8 @@ module Gitlab import_data = project.build_or_assign_import_data( data: { optional_stages: optional_stages, - timeout_strategy: user_settings[:timeout_strategy] + timeout_strategy: user_settings[:timeout_strategy], + extended_events: user_settings[:extended_events] }, credentials: project.import_data&.credentials ) @@ -77,6 +83,10 @@ module Gitlab !enabled?(stage_name) end + def extended_events? + !!project.import_data&.data&.dig('extended_events') + end + private attr_reader :project diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 3584288da57..d4d9bd47e63 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -75,7 +75,7 @@ module Gitlab batch.each do |parent_record| # The page counter needs to be scoped by parent_record to avoid skipping # pages of notes from already imported parent_record. - page_counter = PageCounter.new(project, page_counter_id(parent_record)) + page_counter = Gitlab::Import::PageCounter.new(project, page_counter_id(parent_record)) repo = project.import_source options = collection_options.merge(page: page_counter.current) @@ -85,6 +85,7 @@ module Gitlab yield parent_record, page end + after_batch_processed(parent_record) mark_parent_imported(parent_record) end end @@ -96,6 +97,8 @@ module Gitlab ) end + def after_batch_processed(_parent); end + def already_imported_parents Gitlab::Cache::Import::Caching.values_from_set(parent_imported_cache_key) end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 4bf2d8a0aca..bec4c7fc4d4 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -12,21 +12,18 @@ module Gitlab # Lookups are cached even if no ID was found to remove the need for querying # the database when most queries are not going to return results anyway. class UserFinder + include Gitlab::ExclusiveLeaseHelpers + attr_reader :project, :client - # The base cache key to use for caching user IDs for a given GitHub user - # ID. + # The base cache key to use for caching user IDs for a given GitHub user ID. ID_CACHE_KEY = 'github-import/user-finder/user-id/%s' - # The base cache key to use for caching user IDs for a given GitHub email - # address. - ID_FOR_EMAIL_CACHE_KEY = - 'github-import/user-finder/id-for-email/%s' + # The base cache key to use for caching user IDs for a given GitHub email address. + ID_FOR_EMAIL_CACHE_KEY = 'github-import/user-finder/id-for-email/%s' - # The base cache key to use for caching the Email addresses of GitHub - # usernames. - EMAIL_FOR_USERNAME_CACHE_KEY = - 'github-import/user-finder/email-for-username/%s' + # The base cache key to use for caching the Email addresses of GitHub usernames. + EMAIL_FOR_USERNAME_CACHE_KEY = 'github-import/user-finder/email-for-username/%s' # The base cache key to use for caching the user ETAG response headers USERNAME_ETAG_CACHE_KEY = 'github-import/user-finder/user-etag/%s' @@ -218,6 +215,17 @@ module Gitlab private + def lease_key + "gitlab:github_import:user_finder:#{project.id}" + end + + # Retrieves the email associated with the given username from the cache. + # + # The return value can be an email, an empty string, or nil. + # + # If an empty string is returned, it indicates that the user's email was fetched but not set on GitHub. + # If nil is returned, it indicates that the user's email wasn't fetched or the cache has expired. + # If an email is returned, it means the user has a public email set, and it has been successfully cached. def read_email_from_cache(username) Gitlab::Cache::Import::Caching.read(email_cache_key(username)) end @@ -232,12 +240,27 @@ module Gitlab end def fetch_email_from_github(username, etag: nil) - log(EMAIL_API_CALL_LOGGING_MESSAGE[etag.present?], username: username) - user = client.user(username, { headers: { 'If-None-Match' => etag }.compact }) + in_lock(lease_key, ttl: 3.minutes, sleep_sec: 1.second, retries: 30) do |retried| + # when retried, check the cache again as the other process that had the lease may have fetched the email + if retried + email = read_email_from_cache(username) - user[:email] || '' if user + next email if email.present? + end + + log(EMAIL_API_CALL_LOGGING_MESSAGE[etag.present?], username: username) + + # Only make a rate-limited API call if the ETAG is not available }) + user = client.user(username, { headers: { 'If-None-Match' => etag }.compact }) + user[:email] || '' if user + end end + # Caches the email associated to the username + # + # An empty email is cached when the user email isn't set on GitHub. + # This is done to prevent UserFinder from fetching the user's email again when the user's email isn't set on + # GitHub def cache_email!(username, email) return unless email @@ -245,6 +268,8 @@ module Gitlab end def cache_etag!(username) + return unless client.octokit.last_response + etag = client.octokit.last_response.headers[:etag] Gitlab::Cache::Import::Caching.write(etag_cache_key(username), etag) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index caf7cfb3f76..c4c0e48be3f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -123,7 +123,7 @@ module Gitlab end def add_browsersdk_tracking - return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && Feature.enabled?(:gl_analytics_tracking, + return unless Gitlab.com? && Feature.enabled?(:gl_analytics_tracking, Feature.current_request) return if ENV['GITLAB_ANALYTICS_URL'].blank? || ENV['GITLAB_ANALYTICS_ID'].blank? diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 1a85c57e6b1..40bca7993ce 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -2,9 +2,9 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, language: nil, plain: false, context: {}) + def self.highlight(blob_name, blob_content, language: nil, plain: false, context: {}, used_on: :blob) new(blob_name, blob_content, language: language) - .highlight(blob_content, continue: false, plain: plain, context: context) + .highlight(blob_content, continue: false, plain: plain, context: context, used_on: used_on) end def self.too_large?(size) @@ -18,15 +18,19 @@ module Gitlab @language = language @blob_name = blob_name @blob_content = blob_content + @gitlab_highlight_usage_counter = Gitlab::Metrics.counter( + :gitlab_highlight_usage, + 'The number of times Gitlab::Highlight is used' + ) end - def highlight(text, continue: false, plain: false, context: {}) + def highlight(text, continue: false, plain: false, context: {}, used_on: :blob) @context = context plain ||= self.class.too_large?(text.length) - highlighted_text = highlight_text(text, continue: continue, plain: plain) - highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text = highlight_text(text, continue: continue, plain: plain, used_on: used_on) + highlighted_text = link_dependencies(text, highlighted_text, used_on: used_on) if blob_name highlighted_text end @@ -54,7 +58,9 @@ module Gitlab Rouge::Lexer.find_fancy(@language) end - def highlight_text(text, continue: true, plain: false) + def highlight_text(text, continue: true, plain: false, used_on: :blob) + @gitlab_highlight_usage_counter.increment(used_on: used_on) + if plain highlight_plain(text) else @@ -77,8 +83,8 @@ module Gitlab highlight_plain(text) end - def link_dependencies(text, highlighted_text) - Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + def link_dependencies(text, highlighted_text, used_on: :blob) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text, used_on: used_on) end end end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 9c9816e142e..958b415e18f 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -26,13 +26,6 @@ module Gitlab }.freeze DEFAULT_READ_TOTAL_TIMEOUT = 30.seconds - SILENT_MODE_ALLOWED_METHODS = [ - Net::HTTP::Get, - Net::HTTP::Head, - Net::HTTP::Options, - Net::HTTP::Trace - ].freeze - # We are explicitly assigning these constants because they are used in the codebase. Error = HTTParty::Error Response = HTTParty::Response @@ -42,11 +35,7 @@ module Gitlab class << self ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.each do |method| define_method(method) do |path, options = {}, &block| - if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request) - ::Gitlab::HTTP_V2.public_send(method, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend - else - ::Gitlab::LegacyHTTP.public_send(method, path, options, &block) # rubocop:disable GitlabSecurity/PublicSend - end + ::Gitlab::HTTP_V2.public_send(method, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend -- method is validated to make sure it is one of the methods in Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS end end @@ -59,18 +48,14 @@ module Gitlab # TODO: This method is subject to be removed # We have this for now because we explicitly use the `perform_request` method in some places. def perform_request(http_method, path, options, &block) - if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request) - method_name = http_method::METHOD.downcase.to_sym - - unless ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.include?(method_name) - raise ArgumentError, "Unsupported HTTP method: '#{method_name}'." - end + method_name = http_method::METHOD.downcase.to_sym - # Use `::Gitlab::HTTP_V2.get/post/...` methods - ::Gitlab::HTTP_V2.public_send(method_name, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend - else - ::Gitlab::LegacyHTTP.perform_request(http_method, path, options, &block) + unless ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.include?(method_name) + raise ArgumentError, "Unsupported HTTP method: '#{method_name}'." end + + # Use `::Gitlab::HTTP_V2.get/post/...` methods + ::Gitlab::HTTP_V2.public_send(method_name, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend -- method is validated to make sure it is one of the methods in Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS end private diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 02afdedb4be..d2f916fb02a 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 28, - 'de' => 95, + 'da_DK' => 27, + 'de' => 93, 'en' => 100, 'eo' => 0, - 'es' => 28, + 'es' => 27, 'fil_PH' => 0, - 'fr' => 99, + 'fr' => 97, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 98, - 'ko' => 23, + 'ja' => 94, + 'ko' => 22, 'nb_NO' => 20, 'nl_NL' => 0, - 'pl_PL' => 3, - 'pt_BR' => 60, - 'ro_RO' => 74, - 'ru' => 21, + 'pl_PL' => 2, + 'pt_BR' => 59, + 'ro_RO' => 72, + 'ru' => 20, 'si_LK' => 11, 'tr_TR' => 8, 'uk' => 51, - 'zh_CN' => 99, + 'zh_CN' => 97, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 97 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/github_import/page_counter.rb b/lib/gitlab/import/page_counter.rb index c238ccb8932..44ab9c256d7 100644 --- a/lib/gitlab/github_import/page_counter.rb +++ b/lib/gitlab/import/page_counter.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module GithubImport + module Import # PageCounter can be used to keep track of the last imported page of a # collection, allowing workers to resume where they left off in the event of # an error. @@ -12,7 +12,7 @@ module Gitlab CACHE_KEY = '%{import_type}/page-counter/%{object}/%{collection}' def initialize(object, collection, import_type = 'github-importer') - @cache_key = CACHE_KEY % { import_type: import_type, object: object.id, collection: collection } + @cache_key = format(CACHE_KEY, import_type: import_type, object: object.id, collection: collection) end # Sets the page number to the given value. diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index 5453792e904..ae47c95036e 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -84,7 +84,8 @@ module Gitlab source: 'process_relation_item!', relation_key: relation_key, relation_index: relation_index, - exception: e) + exception: e, + external_identifiers: external_identifiers(data_hash)) end def save_relation_object(relation_object, relation_key, relation_definition, relation_index) @@ -314,6 +315,10 @@ module Gitlab def importable_column_name @column_name ||= @importable.class.reflect_on_association(:import_failures).foreign_key.to_sym end + + def external_identifiers(data_hash) + { iid: data_hash['iid'] }.compact + end end end end diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb index bf7200726a1..f93822a9d95 100644 --- a/lib/gitlab/import_export/import_failure_service.rb +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -13,7 +13,7 @@ module Gitlab end def with_retry(action:, relation_key: nil, relation_index: nil) - on_retry = -> (exception, retry_count, *_args) do + on_retry = ->(exception, retry_count, *_args) do log_import_failure( source: action, relation_key: relation_key, @@ -27,7 +27,8 @@ module Gitlab end end - def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0) + def log_import_failure( + source:, exception:, relation_key: nil, relation_index: nil, retry_count: 0, external_identifiers: {}) attributes = { relation_index: relation_index, source: source, @@ -45,7 +46,8 @@ module Gitlab exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, - relation_key: relation_key + relation_key: relation_key, + external_identifiers: external_identifiers ) ) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index e38930ed548..d34cab5ac92 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1054,6 +1054,7 @@ excluded_attributes: - :state_id - :start_date_sourcing_epic_id - :due_date_sourcing_epic_id + - :issue_id epic_issue: - :epic_id - :issue_id diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 590153ad9cd..3bc9f09d977 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -33,6 +33,10 @@ module Gitlab super.merge(*STORAGES.flat_map(&:payload)) end + def storage_hash + @storage_hash ||= STORAGES.index_by { |k| k.name.demodulize } + end + def detail_store STORAGES.flat_map do |storage| storage.detail_store.map { |details| details.merge(storage: storage.name.demodulize) } diff --git a/lib/gitlab/instrumentation/redis_client_middleware.rb b/lib/gitlab/instrumentation/redis_client_middleware.rb new file mode 100644 index 00000000000..a49d8370d4c --- /dev/null +++ b/lib/gitlab/instrumentation/redis_client_middleware.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# This module references https://github.com/redis-rb/redis-client#instrumentation-and-middlewares +# implementing `call`, and `call_pipelined`. +module Gitlab + module Instrumentation + module RedisClientMiddleware + include RedisHelper + + def call(command, redis_config) + instrumentation = instrumentation_class(redis_config) + + result = instrument_call([command], instrumentation) do + super + end + + measure_io(command, result, instrumentation) if ::RequestStore.active? + + result + end + + def call_pipelined(commands, redis_config) + instrumentation = instrumentation_class(redis_config) + + result = instrument_call(commands, instrumentation, true) do + super + end + + measure_io(commands, result, instrumentation) if ::RequestStore.active? + + result + end + + private + + def measure_io(command, result, instrumentation) + measure_write_size(command, instrumentation) + measure_read_size(result, instrumentation) + end + + def instrumentation_class(config) + Gitlab::Instrumentation::Redis.storage_hash[config.custom[:instrumentation_class]] + end + end + end +end diff --git a/lib/gitlab/instrumentation/redis_helper.rb b/lib/gitlab/instrumentation/redis_helper.rb index ba1c8132250..392a7ebe852 100644 --- a/lib/gitlab/instrumentation/redis_helper.rb +++ b/lib/gitlab/instrumentation/redis_helper.rb @@ -15,7 +15,7 @@ module Gitlab end yield - rescue ::Redis::BaseError => ex + rescue ::Redis::BaseError, ::RedisClient::Error => ex if ex.message.start_with?('MOVED', 'ASK') instrumentation_class.instance_count_cluster_redirection(ex) else diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 8fd8354e59c..a57edcc7ba4 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -23,7 +23,7 @@ module Gitlab def gitlab_id return @gitlab_id if defined?(@gitlab_id) - @gitlab_id = find_by_external_uid || find_by_email + @gitlab_id = find_by_email end private @@ -45,14 +45,6 @@ module Gitlab User.find_by_any_email(email) .try(:id) end - - # rubocop: disable CodeReuse/ActiveRecord - def find_by_external_uid - return unless id - - User.by_provider_and_extern_uid(:github, id).select(:id).first&.id - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb deleted file mode 100644 index cf6ab80d37f..00000000000 --- a/lib/gitlab/legacy_http.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -# -# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686), -# no additional change should be implemented in this class. This class will be removed after migrating all -# the usages to the new gem. -# - -require_relative 'http_connection_adapter' - -module Gitlab - class LegacyHTTP # rubocop:disable Gitlab/NamespacedClass - include HTTParty # rubocop:disable Gitlab/HTTParty - - class << self - alias_method :httparty_perform_request, :perform_request - end - - connection_adapter ::Gitlab::HTTPConnectionAdapter - - def self.perform_request(http_method, path, options, &block) - raise_if_blocked_by_silent_mode(http_method) - - log_info = options.delete(:extra_log_info) - options_with_timeouts = - if !options.has_key?(:timeout) - options.with_defaults(Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS) - else - options - end - - return httparty_perform_request(http_method, path, options_with_timeouts, &block) if options[:stream_body] - - start_time = nil - read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT) - - httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| - start_time ||= ::Gitlab::Metrics::System.monotonic_time - elapsed = ::Gitlab::Metrics::System.monotonic_time - start_time - - if elapsed > read_total_timeout - raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds" - end - - yield fragment if block - end - rescue HTTParty::RedirectionTooDeep - raise Gitlab::HTTP::RedirectionTooDeep - rescue *Gitlab::HTTP::HTTP_ERRORS => e - extra_info = log_info || {} - extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call) - Gitlab::ErrorTracking.log_exception(e, extra_info) - raise e - end - - def self.try_get(path, options = {}, &block) - self.get(path, options, &block) # rubocop:disable Style/RedundantSelf - rescue *Gitlab::HTTP::HTTP_ERRORS - nil - end - - def self.raise_if_blocked_by_silent_mode(http_method) - return unless blocked_by_silent_mode?(http_method) - - ::Gitlab::SilentMode.log_info( - message: 'Outbound HTTP request blocked', - outbound_http_request_method: http_method.to_s - ) - - raise Gitlab::HTTP::SilentModeBlockedError, - 'only get, head, options, and trace methods are allowed in silent mode' - end - - def self.blocked_by_silent_mode?(http_method) - ::Gitlab::SilentMode.enabled? && Gitlab::HTTP::SILENT_MODE_ALLOWED_METHODS.exclude?(http_method) - end - end -end diff --git a/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb index 47ed608c576..9da662d5f1b 100644 --- a/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb +++ b/lib/gitlab/memory/watchdog/handlers/sidekiq_handler.rb @@ -18,8 +18,8 @@ module Gitlab return true unless @alive # Tell sidekiq to restart itself - # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL - send_signal(:TERM, $$, 'gracefully shut down', Sidekiq[:timeout] + 2) + # Keep extra safe to wait `Sidekiq.default_configuration[:timeout] + 2` seconds before SIGKILL + send_signal(:TERM, $$, 'gracefully shut down', Sidekiq.default_configuration[:timeout] + 2) return true unless @alive # Ideally we should never reach this condition diff --git a/lib/gitlab/middleware/unauthenticated_session_expiry.rb b/lib/gitlab/middleware/unauthenticated_session_expiry.rb new file mode 100644 index 00000000000..7c5c523c287 --- /dev/null +++ b/lib/gitlab/middleware/unauthenticated_session_expiry.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + # By default, all sessions are given the same expiration time configured in + # the session store (e.g. 1 week). However, unauthenticated users can + # generate a lot of sessions, primarily for CSRF verification. It makes + # sense to reduce the TTL for unauthenticated to something much lower than + # the default (e.g. 2 hours) to limit Redis memory. In addition, Rails + # creates a new session after login, so the short TTL doesn't even need to + # be extended. + class UnauthenticatedSessionExpiry + def initialize(app) + @app = app + end + + def call(env) + result = @app.call(env) + + warden = env['warden'] + user = catch(:warden) { warden && warden.user } # rubocop:disable Cop/BanCatchThrow -- ignore Warden errors since we're outside Warden::Manager + + unless user + # This works because Rack uses these options every time a request is handled, and redis-store + # uses the Rack setting first: + # 1. https://github.com/rack/rack/blob/fdcd03a3c5a1c51d1f96fc97f9dfa1a9deac0c77/lib/rack/session/abstract/id.rb#L342 + # 2. https://github.com/redis-store/redis-store/blob/3acfa95f4eb6260c714fdb00a3d84be8eedc13b2/lib/redis/store/ttl.rb#L32 + env['rack.session.options'][:expire_after] = Settings.gitlab['unauthenticated_session_expire_delay'] + end + + result + end + end + end +end diff --git a/lib/gitlab/namespaced_session_store.rb b/lib/gitlab/namespaced_session_store.rb index f0f24c081c3..957e8fe9b9f 100644 --- a/lib/gitlab/namespaced_session_store.rb +++ b/lib/gitlab/namespaced_session_store.rb @@ -2,10 +2,8 @@ module Gitlab class NamespacedSessionStore - delegate :[], :[]=, to: :store - def initialize(key, session = Session.current) - @key = key + @namespace_key = key @session = session end @@ -13,11 +11,17 @@ module Gitlab !session.nil? end - def store + def [](key) + return unless session + + session[@namespace_key]&.fetch(key, nil) + end + + def []=(key, value) return unless session - session[@key] ||= {} - session[@key] + session[@namespace_key] ||= {} + session[@namespace_key][key] = value end private diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index cbd523389d6..ebed7bd94e4 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -34,18 +34,18 @@ module Gitlab elsif ordered_by_primary_key? primary_key_order # Ordered by one non-primary table column. Ex. 'ORDER BY created_at'. - elsif ordered_by_other_column? - column_with_tie_breaker_order + elsif ordered_by_other_columns? + columns_with_tie_breaker_order(order_values) # Ordered by two table columns with the last column as a tie breaker. Ex. 'ORDER BY created, id ASC'. - elsif ordered_by_other_column_with_tie_breaker? - tie_breaker_attribute = order_values.second + elsif ordered_by_other_columns_with_tie_breaker? + tie_breaker_attribute = order_values.last tie_breaker_column_order = Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: model_class.primary_key, order_expression: tie_breaker_attribute ) - column_with_tie_breaker_order(tie_breaker_column_order) + columns_with_tie_breaker_order(order_values[0...-1], tie_breaker_column_order) end order ? [scope.reorder!(order), true] : [scope, false] # [scope, success] @@ -120,10 +120,12 @@ module Gitlab ]) end - def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) + def columns_with_tie_breaker_order(order_values, tie_breaker_column_order = default_tie_breaker_column_order) + other_columns = order_values.map { |order_value| column(order_value) } + Gitlab::Pagination::Keyset::Order.build( [ - column(order_values.first), + *other_columns, tie_breaker_column_order ]) end @@ -175,21 +177,27 @@ module Gitlab attribute && primary_key?(attribute) end - def ordered_by_other_column? - return unless order_values.one? + def ordered_by_other_columns? + return unless order_values.size >= 1 && !has_tie_breaker? - supported_column?(order_values.first) + supported_columns?(order_values) end - def ordered_by_other_column_with_tie_breaker? - return unless order_values.size == 2 + def ordered_by_other_columns_with_tie_breaker? + return unless order_values.size >= 2 && supported_columns?(order_values[0...-1]) - return unless supported_column?(order_values.first) + has_tie_breaker? + end - tie_breaker_attribute = order_values.second.try(:expr) + def has_tie_breaker? + tie_breaker_attribute = order_values.last.try(:expr) tie_breaker_attribute && primary_key?(tie_breaker_attribute) end + def supported_columns?(order_values) + order_values.all? { |order_value| supported_column?(order_value) } + end + def default_tie_breaker_column_order Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: model_class.primary_key, diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb index 3f962c47ae9..8c2d1181611 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -7,7 +7,7 @@ require 'sidekiq/version' require 'sidekiq/cron/version' -if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12') +if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('7.1.6') raise 'New version of sidekiq detected, please remove or update this patch' end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index c6a7a39a943..72f4a101809 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -142,9 +142,11 @@ module Gitlab output = "`/#{matched_text[:cmd]}#{" " + matched_text[:arg] if matched_text[:arg]}`" output += "\n" if matched_text[0].include?("\n") elsif keep_actions - # requires an additional newline so that when rendered, it appears - # on its own line, rather than all on the same line - output = "\n#{matched_text[0]}\n" + # put the command in a new paragraph, but without introducing newlines + # so that each command is in its own line, while also preserving sourcemaps + # of the content that follows. + output = ActionController::Base.helpers.simple_format(matched_text[0].chomp) + output += "\n" if matched_text[0].ends_with?("\n") end end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 2f7fa89019e..e4b195767ea 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -197,12 +197,12 @@ module Gitlab @updates[:subscription_event] = 'unsubscribe' end - desc { _('Toggle emoji award') } + desc { _('Toggle emoji reaction') } explanation do |name| - _("Toggles :%{name}: emoji award.") % { name: name } if name + _("Toggles :%{name}: emoji reaction.") % { name: name } if name end execution_message do |name| - _("Toggled :%{name}: emoji award.") % { name: name } if name + _("Toggled :%{name}: emoji reaction.") % { name: name } if name end params ':emoji:' types ::Issuable @@ -213,7 +213,7 @@ module Gitlab match = emoji_param.match(Banzai::Filter::EmojiFilter.emoji_pattern) match[1] if match end - command :award, :react do |name| + command :react, :award do |name| if name && quick_action_target.user_can_award?(current_user) @updates[:emoji_award] = name end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index c79432f36cc..b3f56e8590a 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -240,6 +240,26 @@ module Gitlab @execution_message[:invite_email] = response.message end + desc { _('Remove email participant(s)') } + explanation { _('Removes email participant(s).') } + params 'email1@example.com email2@example.com (up to 6 emails)' + types Issue + condition do + quick_action_target.persisted? && + Feature.enabled?(:issue_email_participants, parent) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) && + quick_action_target.issue_email_participants.any? + end + command :remove_email do |emails = ""| + response = ::IssueEmailParticipants::DestroyService.new( + target: quick_action_target, + current_user: current_user, + emails: emails.split(' ') + ).execute + + @execution_message[:remove_email] = response.message + end + desc { _('Promote issue to incident') } explanation { _('Promotes issue to incident') } execution_message { _('Issue has been promoted to incident') } diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index fe18bc8e133..4276091251a 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -6,6 +6,10 @@ module Gitlab extend ActiveSupport::Concern include Gitlab::QuickActions::Dsl + REBASE_FAILURE_UNMERGEABLE = 'This merge request is currently in an unmergeable state, and cannot be rebased.' + REBASE_FAILURE_PROTECTED_BRANCH = 'This merge request branch is protected from force push.' + REBASE_FAILURE_REBASE_IN_PROGRESS = 'A rebase is already in progress.' + included do # MergeRequest only quick actions definitions desc do @@ -46,6 +50,10 @@ module Gitlab @updates[:merge] = params[:merge_request_diff_head_sha] end + ######################################################################## + # + # /rebase + # types MergeRequest desc do _('Rebase source branch') @@ -66,17 +74,17 @@ module Gitlab end command :rebase do unless quick_action_target.permits_force_push? - @execution_message[:rebase] = _('This merge request branch is protected from force push.') + @execution_message[:rebase] = _(REBASE_FAILURE_PROTECTED_BRANCH) next end if quick_action_target.cannot_be_merged? - @execution_message[:rebase] = _('This merge request cannot be rebased while there are conflicts.') + @execution_message[:rebase] = _(REBASE_FAILURE_UNMERGEABLE) next end if quick_action_target.rebase_in_progress? - @execution_message[:rebase] = _('A rebase is already in progress.') + @execution_message[:rebase] = _(REBASE_FAILURE_REBASE_IN_PROGRESS) next end @@ -210,7 +218,7 @@ module Gitlab explanation { _('Approve the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.eligible_for_approval_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_approval_by?(current_user) && !quick_action_target.merged? end command :approve do success = ::MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) @@ -224,7 +232,7 @@ module Gitlab explanation { _('Unapprove the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.eligible_for_unapproval_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_unapproval_by?(current_user) && !quick_action_target.merged? end command :unapprove do success = ::MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) diff --git a/lib/gitlab/rack_attack/user_allowlist.rb b/lib/gitlab/rack_attack/user_allowlist.rb index f3043f44091..c1da1fabef5 100644 --- a/lib/gitlab/rack_attack/user_allowlist.rb +++ b/lib/gitlab/rack_attack/user_allowlist.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require 'set' +require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. module Gitlab module RackAttack diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 6acbf83df24..f5deff96eae 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -34,7 +34,7 @@ module Gitlab end end - attr_reader :primary_store, :secondary_store, :instance_name + attr_reader :primary_pool, :secondary_pool, :instance_name, :primary_store, :secondary_store, :borrow_counter FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis non_default_store.' @@ -127,20 +127,23 @@ module Gitlab multi ].freeze - # To transition between two Redis store, `primary_store` should be the target store, - # and `secondary_store` should be the current store. Transition is controlled with feature flags: + # To transition between two Redis store, `primary_pool` should be the connection pool for the target store, + # and `secondary_pool` should be the connection pool for the current store. # + # Transition is controlled with feature flags: # - At the default state, all read and write operations are executed in the secondary instance. # - Turning use_primary_and_secondary_stores_for_<instance_name> on: The store writes to both instances. - # The read commands are executed in primary, but fallback to secondary. + # The read commands are executed in the default store with no fallbacks. # Other commands are executed in the the default instance (Secondary). # - Turning use_primary_store_as_default_for_<instance_name> on: The behavior is the same as above, # but other commands are executed in the primary now. # - Turning use_primary_and_secondary_stores_for_<instance_name> off: commands are executed in the primary store. - def initialize(primary_store, secondary_store, instance_name) - @primary_store = primary_store - @secondary_store = secondary_store + def initialize(primary_pool, secondary_pool, instance_name) @instance_name = instance_name + @primary_pool = primary_pool + @secondary_pool = secondary_pool + + @borrow_counter = "multi_store_borrowed_connection_#{instance_name}".to_sym validate_stores! end @@ -208,6 +211,31 @@ module Gitlab true end + def with_borrowed_connection + primary_pool.with do |ps| + secondary_pool.with do |ss| + # nested borrows are allowed as ConnectionPool returns the existing connection + # which the thread already checked out. + Thread.current[borrow_counter] ||= 0 + Thread.current[borrow_counter] += 1 + + # borrow from both pool as feature-flag could change during the period where connections are borrowed + # this guarantees that we avoids a NilClass error + @primary_store = ps + @secondary_store = ss + + yield + ensure + # only set to nil after all nested borrows are yielded + Thread.current[borrow_counter] -= 1 + if Thread.current[borrow_counter] == 0 + @primary_store = nil + @secondary_store = nil + end + end + end + end + # This is needed because of Redis::Rack::Connection is requiring Redis::Store # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 @@ -277,13 +305,15 @@ module Gitlab # # Let's define it explicitly instead of propagating it to method_missing def close - if same_redis_store? - # if same_redis_store?, `use_primary_store_as_default?` returns false - # but we should avoid a feature-flag check in `.close` to avoid checking out - # an ActiveRecord connection during clean up. - secondary_store.close - else - [primary_store, secondary_store].map(&:close).first + with_borrowed_connection do + if same_redis_store? + # if same_redis_store?, `use_primary_store_as_default?` returns false + # but we should avoid a feature-flag check in `.close` to avoid checking out + # an ActiveRecord connection during clean up. + secondary_store.close + else + [primary_store, secondary_store].map(&:close).first + end end end @@ -383,7 +413,8 @@ module Gitlab def same_redis_store? strong_memoize(:same_redis_store) do # <Redis client v4.7.1 for unix:///path_to/redis/redis.socket/5>" - primary_store.inspect == secondary_store.inspect + # no borrowed connections due to endless recursion + primary_pool.with(&:inspect) == secondary_pool.with(&:inspect) # rubocop:disable CodeReuse/ActiveRecord end end @@ -418,16 +449,16 @@ module Gitlab @instance = nil end - def redis_store?(store) - store.is_a?(::Redis) + def redis_store?(pool) + pool.with { |c| c.instance_of?(Gitlab::Redis::MultiStore) || c.is_a?(::Redis) } end def validate_stores! - raise ArgumentError, 'primary_store is required' unless primary_store - raise ArgumentError, 'secondary_store is required' unless secondary_store + raise ArgumentError, 'primary_store is required' if primary_pool.nil? + raise ArgumentError, 'secondary_store is required' if secondary_pool.nil? raise ArgumentError, 'instance_name is required' unless instance_name - raise ArgumentError, 'invalid primary_store' unless redis_store?(primary_store) - raise ArgumentError, 'invalid secondary_store' unless redis_store?(secondary_store) + raise ArgumentError, 'invalid primary_store' unless redis_store?(primary_pool) + raise ArgumentError, 'invalid secondary_store' unless redis_store?(secondary_pool) end end end diff --git a/lib/gitlab/redis/multi_store_wrapper.rb b/lib/gitlab/redis/multi_store_wrapper.rb new file mode 100644 index 00000000000..093a06f2737 --- /dev/null +++ b/lib/gitlab/redis/multi_store_wrapper.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class MultiStoreWrapper < Wrapper + class << self + def with + multistore_pool.with do |multistore| + multistore.with_borrowed_connection do + yield multistore + end + end + end + + def multistore_pool + @multistore_pool ||= ConnectionPool.new(size: pool_size, name: pool_name) { multistore } + end + + def pool_name + "#{store_name}MultiStore".underscore + end + + def multistore + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index d12d3e8c6aa..e631bfaa01c 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -2,12 +2,9 @@ module Gitlab module Redis - class SharedState < ::Gitlab::Redis::Wrapper - def self.redis - primary_store = ::Redis.new(ClusterSharedState.params) - secondary_store = ::Redis.new(params) - - MultiStore.new(primary_store, secondary_store, store_name) + class SharedState < ::Gitlab::Redis::MultiStoreWrapper + def self.multistore + MultiStore.new(ClusterSharedState.pool, pool, store_name) end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index bb231eec226..1f5d8ab7c9b 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -19,7 +19,7 @@ module Gitlab InvalidPathError = Class.new(StandardError) class << self - delegate :params, :url, :store, :encrypted_secrets, to: :new + delegate :params, :url, :store, :encrypted_secrets, :redis_client_params, to: :new def with pool.with { |redis| yield redis } @@ -30,7 +30,12 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size, name: store_name.underscore) { redis } + @pool ||= if config_fallback && + config_fallback.params.except(:instrumentation_class) == params.except(:instrumentation_class) + config_fallback.pool + else + ConnectionPool.new(size: pool_size, name: store_name.underscore) { redis } + end end def pool_size @@ -96,6 +101,33 @@ module Gitlab redis_store_options end + # redis_client_params modifies redis_store_options to be compatible with redis-client + # TODO: when redis-rb is updated to v5, there is no need to support 2 types of config format + def redis_client_params + options = redis_store_options + + # avoid passing classes into options as Sidekiq scrubs the options with Marshal.dump + Marshal.load + # ref https://github.com/sidekiq/sidekiq/blob/v7.1.6/lib/sidekiq/redis_connection.rb#L37 + # + # this does not play well with spring enabled as the forked process references the old constant + # we use strings to look up Gitlab::Instrumentation::Redis.storage_hash as a bypass + options[:custom] = { instrumentation_class: self.class.store_name } + + # TODO: add support for cluster when upgrading to redis-rb v5.y.z we do not need cluster support + # as Sidekiq workload should not and does not run in a Redis Cluster + # support to be added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134862 + if options[:sentinels] + # name is required in RedisClient::SentinelConfig + # https://github.com/redis-rb/redis-client/blob/1ab081c1d0e47df5d55e011c9390c70b2eef6731/lib/redis_client/sentinel_config.rb#L17 + options[:name] = options[:host] + options.except(:scheme, :instrumentation_class, :host, :port) + else + # remove disallowed keys as seen in + # https://github.com/redis-rb/redis-client/blob/1ab081c1d0e47df5d55e011c9390c70b2eef6731/lib/redis_client/config.rb#L21 + options.except(:scheme, :instrumentation_class) + end + end + def url raw_config_hash[:url] end @@ -183,6 +215,7 @@ module Gitlab config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) + redis_hash[:ssl] = true if redis_hash[:scheme] == 'rediss' # order is important here, sentinels must be after the connection keys. # {url: ..., port: ..., sentinels: [...]} redis_hash.merge(config) diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 269fb74ceca..e560db7ace8 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -89,12 +89,11 @@ module Gitlab if puma? && ::Puma.respond_to?(:cli_config) threads += ::Puma.cli_config.options[:max_threads] elsif sidekiq? - # 2 extra threads for the pollers in Sidekiq and Sidekiq Cron: - # https://github.com/ondrejbartas/sidekiq-cron#under-the-hood + # Sidekiq has a internal connection pool to handle heartbeat, scheduled polls, + # cron polls and housekeeping. max_threads can match Sidekqi process's concurrency. # - # These threads execute Sidekiq client middleware when jobs - # are enqueued and those can access DB / Redis. - threads += Sidekiq[:concurrency] + 2 + # The Sidekiq main thread does not perform GitLab-related logic, so we can ignore it. + threads = Sidekiq.default_configuration[:concurrency] end if puma? diff --git a/lib/gitlab/security/features.rb b/lib/gitlab/security/features.rb new file mode 100644 index 00000000000..2176e588d77 --- /dev/null +++ b/lib/gitlab/security/features.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +module Gitlab + module Security + class Features + # rubocop: disable Metrics/AbcSize -- Generate dynamic translation as per + # https://docs.gitlab.com/ee/development/i18n/externalization.html#keep-translations-dynamic + def self.data + { + sast: { + name: _('Static Application Security Testing (SAST)'), + short_name: _('SAST'), + description: _('Analyze your source code for known vulnerabilities.'), + help_path: Gitlab::Routing.url_helpers.help_page_path('user/application_security/sast/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path('user/application_security/sast/index', + anchor: 'configuration'), + type: 'sast' + }, + sast_iac: { + name: _('Infrastructure as Code (IaC) Scanning'), + short_name: s_('ciReport|SAST IaC'), + description: _('Analyze your infrastructure as code configuration files for known vulnerabilities.'), + help_path: Gitlab::Routing.url_helpers.help_page_path('user/application_security/iac_scanning/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/iac_scanning/index', + anchor: 'configuration'), + type: 'sast_iac' + }, + dast: { + badge: { + text: _('Available on demand'), + tooltip_text: _( + 'On-demand scans run outside of the DevOps cycle and find vulnerabilities in your projects'), + variant: 'info' + }, + secondary: { + type: 'dast_profiles', + name: _('DAST profiles'), + description: s_('SecurityConfiguration|Manage profiles for use by DAST scans.'), + configuration_text: s_('SecurityConfiguration|Manage profiles') + }, + name: _('Dynamic Application Security Testing (DAST)'), + short_name: s_('ciReport|DAST'), + description: s_('ciReport|Analyze a deployed version of your web application for known ' \ + 'vulnerabilities by examining it from the outside in. DAST works ' \ + 'by simulating external attacks on your application while it is running.'), + help_path: Gitlab::Routing.url_helpers.help_page_path('user/application_security/dast/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path('user/application_security/dast/index', + anchor: 'enable-automatic-dast-run'), + type: 'dast', + anchor: 'dast' + }, + dependency_scanning: { + name: _('Dependency Scanning'), + description: _('Analyze your dependencies for known vulnerabilities.'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/dependency_scanning/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/dependency_scanning/index', anchor: 'configuration'), + type: 'dependency_scanning', + anchor: 'dependency-scanning' + }, + container_scanning: { + name: _('Container Scanning'), + description: _('Check your Docker images for known vulnerabilities.'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/container_scanning/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/container_scanning/index', anchor: 'configuration'), + type: 'container_scanning' + }, + secret_detection: { + name: _('Secret Detection'), + description: _('Analyze your source code and git history for secrets.'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/secret_detection/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/secret_detection/index', anchor: 'configuration'), + type: 'secret_detection' + }, + api_fuzzing: { + name: _('API Fuzzing'), + description: _('Find bugs in your code with API fuzzing.'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/api_fuzzing/index'), + type: 'api_fuzzing' + }, + coverage_fuzzing: { + name: _('Coverage Fuzzing'), + description: _('Find bugs in your code with coverage-guided fuzzing.'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/coverage_fuzzing/index'), + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/coverage_fuzzing/index', anchor: 'enable-coverage-guided-fuzz-testing'), + type: 'coverage_fuzzing', + secondary: { + type: 'corpus_management', + name: _('Corpus Management'), + description: s_('SecurityConfiguration|Manage corpus files used as seed ' \ + 'inputs with coverage-guided fuzzing.'), + configuration_text: s_('SecurityConfiguration|Manage corpus') + } + }, + breach_and_attack_simulation: { + anchor: 'bas', + badge: { + always_display: true, + text: s_('SecurityConfiguration|Incubating feature'), + tooltip_text: s_('SecurityConfiguration|Breach and Attack Simulation is an incubating ' \ + 'feature extending existing security testing by simulating adversary activity.'), + variant: 'info' + }, + description: s_('SecurityConfiguration|Simulate breach and attack scenarios against your ' \ + 'running application by attempting to detect and exploit known vulnerabilities.'), + name: s_('SecurityConfiguration|Breach and Attack Simulation (BAS)'), + help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/breach_and_attack_simulation/index'), + secondary: { + configuration_help_path: Gitlab::Routing.url_helpers.help_page_path( + 'user/application_security/breach_and_attack_simulation/index', + anchor: 'extend-dynamic-application-security-testing-dast'), + description: s_('SecurityConfiguration|Enable incubating Breach and Attack Simulation focused ' \ + 'features such as callback attacks in your DAST scans.'), + name: s_('SecurityConfiguration|Out-of-Band Application Security Testing (OAST)') + }, + short_name: s_('SecurityConfiguration|BAS'), + type: 'breach_and_attack_simulation' + } + }.freeze + end + # rubocop: enable Metrics/AbcSize + end + end +end diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 18767dd332a..c5faf3f589f 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -37,6 +37,10 @@ module Gitlab false end + def security_features + Features.data[type] || {} + end + private attr_reader :project, :configured diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index b2ff80b2357..f4dabb7498f 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'yaml' +require 'sidekiq/capsule' module Gitlab module SidekiqConfig @@ -161,7 +162,7 @@ module Gitlab # the current Sidekiq process def current_worker_queue_mappings worker_queue_mappings - .select { |worker, queue| Sidekiq[:queues].include?(queue) } + .select { |worker, queue| Sidekiq.default_configuration.queues.include?(queue) } .to_h end diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index c49180a6c1c..5c69a87f366 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'yaml' -require 'set' +require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. # These methods are called by `sidekiq-cluster`, which runs outside of # the bundler/Rails context, so we cannot use any gem or Rails methods. diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index c65d9c5ddd5..4754417639f 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -16,11 +16,11 @@ module Gitlab ActiveRecord::LogSubscriber.reset_runtime - Sidekiq.logger.info log_job_start(job, base_payload) + @logger.info log_job_start(job, base_payload) yield - Sidekiq.logger.info log_job_done(job, started_time, base_payload) + @logger.info log_job_done(job, started_time, base_payload) rescue Sidekiq::JobRetry::Handled => job_exception # Sidekiq::JobRetry::Handled is raised by the internal Sidekiq # processor. It is a wrapper around real exception indicating an @@ -29,11 +29,11 @@ module Gitlab # # For more information: # https://github.com/mperham/sidekiq/blob/v5.2.7/lib/sidekiq/processor.rb#L173 - Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) + @logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) raise rescue StandardError => job_exception - Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) + @logger.warn log_job_done(job, started_time, base_payload, job_exception) raise end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 37a9ed37891..e65761fc1b6 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -64,7 +64,7 @@ module Gitlab def initialize_process_metrics metrics = self.metrics - metrics[:sidekiq_concurrency].set({}, Sidekiq[:concurrency].to_i) + metrics[:sidekiq_concurrency].set({}, Sidekiq.default_configuration[:concurrency].to_i) return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb index 2467dd7ca43..cf4893b8745 100644 --- a/lib/gitlab/sidekiq_migrate_jobs.rb +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -16,17 +16,14 @@ module Gitlab # Migrate jobs in SortedSets, i.e. scheduled and retry sets. def migrate_set(sidekiq_set) source_queues_regex = Regexp.union(mappings.keys) - cursor = 0 scanned = 0 migrated = 0 estimated_size = Sidekiq.redis { |c| c.zcard(sidekiq_set) } logger&.info("Processing #{sidekiq_set} set. Estimated size: #{estimated_size}.") - begin - cursor, jobs = Sidekiq.redis { |c| c.zscan(sidekiq_set, cursor) } - - jobs.each do |(job, score)| + Sidekiq.redis do |c| + c.zscan(sidekiq_set) do |job, score| if scanned > 0 && scanned % LOG_FREQUENCY == 0 logger&.info("In progress. Scanned records: #{scanned}. Migrated records: #{migrated}.") end @@ -45,7 +42,7 @@ module Gitlab migrated += migrate_job_in_set(sidekiq_set, job, score, job_hash) end - end while cursor.to_i != 0 + end logger&.info("Done. Scanned records: #{scanned}. Migrated records: #{migrated}.") @@ -61,7 +58,7 @@ module Gitlab logger&.info("List of queues based on routing rules: #{routing_rules_queues}") Sidekiq.redis do |conn| # Redis 6 supports conn.scan_each(match: "queue:*", type: 'list') - conn.scan_each(match: "queue:*") do |key| + conn.scan("MATCH", "queue:*") do |key| # Redis 5 compatibility next unless conn.type(key) == 'list' @@ -101,13 +98,9 @@ module Gitlab Sidekiq.redis do |connection| removed = connection.zrem(sidekiq_set, job) - if removed - connection.zadd(sidekiq_set, score, Gitlab::Json.dump(job_hash)) + connection.zadd(sidekiq_set, score, Gitlab::Json.dump(job_hash)) if removed > 0 - 1 - else - 0 - end + removed end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 496ed9de828..c25e4e776cd 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -37,7 +37,7 @@ module Gitlab return unless expire with_redis do |redis| - redis.set(key_for(jid), 1, ex: expire) + redis.set(key_for(jid), 1, ex: expire.to_i) end end @@ -56,7 +56,7 @@ module Gitlab # expire - The expiration time of the Redis key. def self.expire(jid, expire = DEFAULT_EXPIRATION) with_redis do |redis| - redis.expire(key_for(jid), expire) + redis.expire(key_for(jid), expire.to_i) end end diff --git a/lib/gitlab/ssh/commit.rb b/lib/gitlab/ssh/commit.rb index 7d7cc529b1a..4ea64e117ac 100644 --- a/lib/gitlab/ssh/commit.rb +++ b/lib/gitlab/ssh/commit.rb @@ -10,7 +10,7 @@ module Gitlab end def attributes - signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, signer, @commit.committer_email) + signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, signer, @commit) { commit_sha: @commit.sha, diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index 6b0cab75557..5f4b6cdcb8d 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -11,11 +11,12 @@ module Gitlab GIT_NAMESPACE = 'git' - def initialize(signature_text, signed_text, signer, committer_email) + def initialize(signature_text, signed_text, signer, commit) @signature_text = signature_text @signed_text = signed_text @signer = signer - @committer_email = committer_email + @commit = commit + @committer_email = commit.committer_email end def verification_status @@ -23,15 +24,8 @@ module Gitlab next :unverified unless all_attributes_present? next :verified_system if verified_by_gitlab? next :unverified unless valid_signature_blob? - next :unknown_key unless signed_by_key - next :other_user unless committer - next :other_user unless signed_by_key.user == committer - - if signed_by_user_email_verified? - :verified - else - :unverified - end + + calculate_verification_status end end @@ -44,15 +38,23 @@ module Gitlab end def key_fingerprint - strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } + strong_memoize(:key_fingerprint) do + public_key = signature&.public_key + + next public_key.public_key.fingerprint if public_key.is_a?(SSHData::Certificate) + + public_key.fingerprint + end end private + attr_reader :commit, :committer_email + def all_attributes_present? # Signing an empty string is valid, but signature_text and committer_email # must be non-empty. - @signed_text && @signature_text.present? && @committer_email.present? + @signed_text && @signature_text.present? && committer_email.present? end # Verifies the signature using the public key embedded in the blob. @@ -66,14 +68,23 @@ module Gitlab signature.verify(@signed_text) end - def committer + def calculate_verification_status + return :unknown_key unless signed_by_key + return :other_user unless committer? + return :unverified unless signed_by_user_email_verified? + + :verified + end + + def committer? # Lookup by email because users can push verified commits that were made # by someone else. For example: Doing a rebase. - strong_memoize(:committer) { User.find_by_any_email(@committer_email) } + committer = User.find_by_any_email(committer_email) + committer && signed_by_key.user == committer end def signed_by_user_email_verified? - signed_by_key.user.verified_emails.include?(@committer_email) + signed_by_key.user.verified_emails.include?(committer_email) end def signature @@ -95,3 +106,5 @@ module Gitlab end end end + +Gitlab::Ssh::Signature.prepend_mod diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 102104644f7..a26b4d5bbcb 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,22 +10,22 @@ module Gitlab APPLICATION_DEFAULT = 3 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) + Theme = Struct.new(:id, :name, :css_class, :primary_color) # All available Themes def available_themes [ - Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', 'theme_indigo', '#222261'), - Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', 'theme_light_indigo', '#41419f'), - Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', 'theme_blue', '#0b2640'), - Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', 'theme_light_blue', '#145aa1'), - Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', 'theme_green', '#0e4328'), - Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#1b653f'), - Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#580d02'), - Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a02e1c'), - Theme.new(2, s_('NavigationTheme|Gray'), 'ui-gray', 'theme_gray', '#333238'), - Theme.new(3, s_('NavigationTheme|Light Gray'), 'ui-light-gray', 'theme_light_gray', '#ececef'), - Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#1f1e24') + Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', '#222261'), + Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', '#41419f'), + Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', '#0b2640'), + Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', '#145aa1'), + Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', '#0e4328'), + Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', '#1b653f'), + Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', '#580d02'), + Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', '#a02e1c'), + Theme.new(2, s_('NavigationTheme|Gray'), 'ui-gray', '#333238'), + Theme.new(3, s_('NavigationTheme|Light Gray'), 'ui-light-gray', '#ececef'), + Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', '#1f1e24') ] end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 0b606b712c7..df10555f006 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -33,21 +33,6 @@ module Gitlab track_struct_event(tracker, category, action, label: label, property: property, value: value, contexts: contexts) end - def database_event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists - action = action.to_s - destination = Gitlab::Tracking::Destinations::DatabaseEventsSnowplow.new - contexts = [ - Tracking::StandardContext.new( - namespace_id: namespace&.id, - plan_name: namespace&.actual_plan_name, - project_id: project&.id, - user_id: user&.id, - **extra).to_context, *context - ] - - track_struct_event(destination, category, action, label: label, property: property, value: value, contexts: contexts) - end - def definition(basename, category: nil, action: nil, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists definition = YAML.load_file(Rails.root.join("config/events/#{basename}.yml")) diff --git a/lib/gitlab/tracking/destinations/database_events_snowplow.rb b/lib/gitlab/tracking/destinations/database_events_snowplow.rb deleted file mode 100644 index 458d7f0c129..00000000000 --- a/lib/gitlab/tracking/destinations/database_events_snowplow.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracking - module Destinations - class DatabaseEventsSnowplow < Snowplow - extend ::Gitlab::Utils::Override - - HOSTNAME = 'db-snowplow.trx.gitlab.net' - - override :enabled? - # database events are only collected for SaaS instance - def enabled? - ::Gitlab.dev_or_test_env? || ::Gitlab.com? - end - - override :hostname - def hostname - return Gitlab::CurrentSettings.snowplow_database_collector_hostname || HOSTNAME if ::Gitlab.com? - - 'localhost:9091' - end - - private - - override :increment_failed_events_emissions - def increment_failed_events_emissions(value) - Gitlab::Metrics.counter( - :gitlab_db_events_snowplow_failed_events_total, - 'Number of failed Snowplow events emissions' - ).increment({}, value.to_i) - end - - override :increment_successful_events_emissions - def increment_successful_events_emissions(value) - Gitlab::Metrics.counter( - :gitlab_db_events_snowplow_successful_events_total, - 'Number of successful Snowplow events emissions' - ).increment({}, value.to_i) - end - - override :increment_total_events_counter - def increment_total_events_counter - Gitlab::Metrics.counter( - :gitlab_db_events_snowplow_events_total, - 'Number of Snowplow events' - ).increment - end - end - end - end -end diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb index 9d197de454e..ce8263e824b 100644 --- a/lib/gitlab/tracking/event_definition.rb +++ b/lib/gitlab/tracking/event_definition.rb @@ -27,7 +27,7 @@ module Gitlab definition = YAML.safe_load(definition) definition.deep_symbolize_keys! - self.new(path, definition).tap(&:validate!) + self.new(path, definition) rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(e.message)) end @@ -51,17 +51,15 @@ module Gitlab path.delete_prefix(Rails.root.to_s) end - def validate! - SCHEMA.validate(attributes.stringify_keys).each do |error| - error_message = <<~ERROR_MSG + def validation_errors + SCHEMA.validate(attributes.stringify_keys).map do |error| + <<~ERROR_MSG + --------------- VALIDATION ERROR --------------- + Definition file: #{path} Error type: #{error['type']} Data: #{error['data']} Path: #{error['data_pointer']} - Details: #{error['details']} - Definition file: #{path} ERROR_MSG - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(error_message)) end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 5eddf8da7dd..71d3680e67e 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -59,17 +59,15 @@ module Gitlab attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end - def validate! - errors.each do |error| - error_message = <<~ERROR_MSG + def validation_errors + errors.map do |error| + <<~ERROR_MSG + --------------- VALIDATION ERROR --------------- + Metric file: #{path} Error type: #{error['type']} Data: #{error['data']} Path: #{error['data_pointer']} - Details: #{error['details']} - Metric file: #{path} ERROR_MSG - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message)) end end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index e0a4f879f48..b8a964be59a 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -30,21 +30,6 @@ module Gitlab::UsageDataCounters Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml'))&.full_name end - def all_included_templates(template_name) - expanded_template_name = expand_template_name(template_name) - results = [expanded_template_name].tap do |result| - template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) - data = Gitlab::Ci::Config::Yaml::Loader.new(template.content).load.content - [data[:include]].compact.flatten.each do |ci_include| - if ci_include_template = ci_include[:template] - result.concat(all_included_templates(ci_include_template)) - end - end - end - - results.uniq.sort_by { _1['name'] } - end - private def template_to_event_name(template) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index b0444066722..137b6f90545 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -8,6 +8,7 @@ module Gitlab module HLLRedisCounter KEY_EXPIRY_LENGTH = 6.weeks REDIS_SLOT = 'hll_counters' + KEY_OVERRIDES_PATH = Rails.root.join('lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml') EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) @@ -22,6 +23,7 @@ module Gitlab include Gitlab::Utils::UsageData include Gitlab::Usage::TimeFrame include Gitlab::Usage::TimeSeriesStorable + include Gitlab::Utils::StrongMemoize # Track unique events # @@ -105,13 +107,23 @@ module Gitlab end def redis_key(event, time) - raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) - - key = "{#{REDIS_SLOT}}_#{event[:name]}" + key = redis_key_base(event[:name]) year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" + "{#{REDIS_SLOT}}_#{key}-#{year_week}" + end + + def redis_key_base(event_name) + raise UnknownEvent, "Unknown event #{event_name}" unless known_events_names.include?(event_name.to_s) + + key_overrides.fetch(event_name, event_name) end + + def key_overrides + YAML.safe_load(File.read(KEY_OVERRIDES_PATH)) + end + + strong_memoize_attr :key_overrides end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml @@ -0,0 +1 @@ +{} diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index d26b7ce951d..db48095ab74 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -6,7 +6,6 @@ module Gitlab MR_DIFFS_ACTION = 'i_code_review_mr_diffs' MR_DIFFS_SINGLE_FILE_ACTION = 'i_code_review_mr_single_file_diffs' MR_DIFFS_USER_SINGLE_FILE_ACTION = 'i_code_review_user_single_file_diffs' - MR_CREATE_ACTION = 'i_code_review_create_mr' MR_USER_CREATE_ACTION = 'i_code_review_user_create_mr' MR_CLOSE_ACTION = 'i_code_review_user_close_mr' MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' @@ -64,15 +63,10 @@ module Gitlab end def track_create_mr_action(user:, merge_request:) - track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) - - project = merge_request.target_project - Gitlab::InternalEvents.track_event( MR_USER_CREATE_ACTION, user: user, - project: project, - namespace: project.namespace + project: merge_request.target_project ) end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index 557179ad57a..6e4c5d4e845 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -19,6 +19,8 @@ module Gitlab def prepare_name(name, args) case name + when 'react' + 'award' when 'assign' event_name_for_assign(args) when 'copy_metadata' @@ -35,6 +37,8 @@ module Gitlab event_name_for_unlabel(args) when 'invite_email' 'invite_email' + event_name_quantifier(args.split) + when 'remove_email' + 'remove_email' + event_name_quantifier(args.split) else name end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index b99c9ebb24f..3a090c0a3d1 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -8,6 +8,7 @@ module Gitlab WORK_ITEM_DATE_CHANGED = 'users_updating_work_item_dates' WORK_ITEM_LABELS_CHANGED = 'users_updating_work_item_labels' WORK_ITEM_MILESTONE_CHANGED = 'users_updating_work_item_milestone' + WORK_ITEM_TODO_MARKED = 'users_updating_work_item_todo' class << self def track_work_item_created_action(author:) @@ -30,6 +31,10 @@ module Gitlab track_unique_action(WORK_ITEM_MILESTONE_CHANGED, author) end + def track_work_item_mark_todo_action(author:) + track_unique_action(WORK_ITEM_TODO_MARKED, author) + end + private def track_unique_action(action, author) diff --git a/lib/integrations/google_cloud_platform/artifact_registry/client.rb b/lib/integrations/google_cloud_platform/artifact_registry/client.rb index ae41aa2614e..32e09821814 100644 --- a/lib/integrations/google_cloud_platform/artifact_registry/client.rb +++ b/lib/integrations/google_cloud_platform/artifact_registry/client.rb @@ -15,11 +15,13 @@ module Integrations end def list_docker_images(page_token: nil) + url = list_docker_images_url response = ::Gitlab::HTTP.get( - list_docker_images_url, + url, headers: headers, query: query_params(page_token: page_token), - format: :plain # disable httparty json parsing + format: :plain, # disable httparty json parsing + extra_allowed_uris: [URI(GLGO_BASE_URL)] ) if response.success? diff --git a/lib/integrations/google_cloud_platform/base_client.rb b/lib/integrations/google_cloud_platform/base_client.rb index 56c05e7987b..937454cda43 100644 --- a/lib/integrations/google_cloud_platform/base_client.rb +++ b/lib/integrations/google_cloud_platform/base_client.rb @@ -6,7 +6,7 @@ module Integrations GLGO_BASE_URL = if Gitlab.staging? 'https://glgo.staging.runway.gitlab.net' else - 'http://glgo.runway.gitlab.net/' + 'https://glgo.runway.gitlab.net' end def initialize(project:, user:) diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index 813ff0bf097..de7b275d43e 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -45,7 +45,7 @@ module Quality created_at += 1.week - break if created_at > Time.now + break if created_at.future? end created_issues_count diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index ece6460bb89..d20dec2f8b9 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -6,28 +6,17 @@ module Sidebars class SettingsMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - if can?(context.current_user, :admin_group, context.group) - add_item(general_menu_item) - add_item(integrations_menu_item) - add_item(access_tokens_menu_item) - add_item(group_projects_menu_item) - add_item(repository_menu_item) - add_item(ci_cd_menu_item) - add_item(applications_menu_item) - add_item(packages_and_registries_menu_item) - add_item(usage_quotas_menu_item) - return true - elsif Gitlab.ee? && can?(context.current_user, :change_push_rules, context.group) - # Push Rules are the only group setting that can also be edited by maintainers. - # Create an empty sub-menu here and EE adds Repository menu item (with only Push Rules). - return true - elsif Gitlab.ee? && can?(context.current_user, :read_billing, context.group) - # Billing is the only group setting that is visible to auditors. - # Create an empty sub-menu here and EE adds Settings menu item (with only Billing). - return true - end - - false + return unless can?(context.current_user, :admin_group, context.group) + + add_item(general_menu_item) + add_item(integrations_menu_item) + add_item(access_tokens_menu_item) + add_item(group_projects_menu_item) + add_item(repository_menu_item) + add_item(ci_cd_menu_item) + add_item(applications_menu_item) + add_item(packages_and_registries_menu_item) + add_item(usage_quotas_menu_item) end override :title diff --git a/lib/sidebars/organizations/menus/scope_menu.rb b/lib/sidebars/organizations/menus/scope_menu.rb index a535be21280..559e57bc171 100644 --- a/lib/sidebars/organizations/menus/scope_menu.rb +++ b/lib/sidebars/organizations/menus/scope_menu.rb @@ -27,7 +27,7 @@ module Sidebars override :serialize_as_menu_item_args def serialize_as_menu_item_args super.merge({ - avatar: nil, + avatar: context.container.avatar_url(size: 48), entity_id: context.container.id, super_sidebar_parent: ::Sidebars::StaticMenu, item_id: :organization_overview diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 5a378d5f9a8..0826349f6d3 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -11,6 +11,7 @@ module Sidebars add_item(infrastructure_registry_menu_item) add_item(harbor_registry_menu_item) add_item(model_experiments_menu_item) + add_item(model_registry_menu_item) true end @@ -47,7 +48,7 @@ module Sidebars end def container_registry_menu_item - if !::Gitlab.config.registry.enabled || !can?(context.current_user, :read_container_image, context.project) + if container_registry_unavailable? return ::Sidebars::NilMenuItem.new(item_id: :container_registry) end @@ -103,11 +104,32 @@ module Sidebars ) end + def model_registry_menu_item + unless can?(context.current_user, :read_model_registry, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :model_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Model registry'), + link: project_ml_models_path(context.project), + super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::DeployMenu, + active_routes: { controller: %w[projects/ml/models] }, + item_id: :model_registry + ) + end + def packages_registry_disabled? !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project&.packages_policy_subject) end + + def container_registry_unavailable? + !::Gitlab.config.registry.enabled || + !can?(context.current_user, :read_container_image, context.project) + end end end end end + +Sidebars::Projects::Menus::PackagesRegistriesMenu.prepend_mod_with('Sidebars::Projects::Menus::PackagesRegistriesMenu') diff --git a/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb index 9f667466d1c..f3f73fc4b78 100644 --- a/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb +++ b/lib/sidebars/projects/super_sidebar_menus/deploy_menu.rb @@ -20,7 +20,8 @@ module Sidebars :releases, :feature_flags, :packages_registry, - :container_registry + :container_registry, + :model_registry ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ef9d2b5e13a..c20190a2f64 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,7 +1,7 @@ # frozen_string_literal: true namespace :gitlab do - require 'set' + require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported. namespace :cleanup do desc "GitLab | Cleanup | Block users that have been removed in LDAP" @@ -51,7 +51,7 @@ namespace :gitlab do end end - desc 'GitLab | Cleanup | Clean orphan job artifact files' + desc 'GitLab | Cleanup | Clean orphan job artifact files in local storage' task orphan_job_artifact_files: :gitlab_environment do warn_user_is_not_gitlab @@ -63,6 +63,31 @@ namespace :gitlab do end end + desc 'GitLab | Cleanup | Clean orphan job artifact files stored in the @final directory in object storage' + task :orphan_job_artifact_final_objects, [:provider] => :gitlab_environment do |_, args| + warn_user_is_not_gitlab + + force_restart = ENV['FORCE_RESTART'].present? + + begin + cleaner = Gitlab::Cleanup::OrphanJobArtifactFinalObjectsCleaner.new( + provider: args.provider, + force_restart: force_restart, + dry_run: dry_run?, + logger: logger + ) + + cleaner.run! + + if dry_run? + logger.info "To clean up all orphan files that were found, run this command with DRY_RUN=false".color(:yellow) + end + rescue Gitlab::Cleanup::OrphanJobArtifactFinalObjectsCleaner::UnsupportedProviderError => e + abort %(#{e.message} +Usage: rake "gitlab:cleanup:orphan_job_artifact_final_objects[provider]") + end + end + desc 'GitLab | Cleanup | Clean orphan LFS file references' task orphan_lfs_file_references: :gitlab_environment do warn_user_is_not_gitlab @@ -136,7 +161,7 @@ namespace :gitlab do next unless latest_diff_sha branches_to_delete << { reference: mr.source_branch_ref, old_sha: latest_diff_sha, -new_sha: Gitlab::Git::BLANK_SHA } +new_sha: Gitlab::Git::SHA1_BLANK_SHA } break if number_deleted + branches_to_delete.size >= limit end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 1cd72ee6a1b..09db25735d7 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -48,39 +48,5 @@ namespace :gitlab do FileUtils.mkdir_p(path) File.write(File.join(path, 'sql_metrics_queries.json'), Gitlab::Json.pretty_generate(queries)) end - - # Events for templates included via YAML-less Auto-DevOps - def implicit_auto_devops_includes - Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .all_included_templates('Auto-DevOps.gitlab-ci.yml') - .map { |template| implicit_auto_devops_event(template) } - .uniq - .sort_by { _1['name'] } - end - - # Events for templates included in a .gitlab-ci.yml using include:template - def explicit_template_includes - Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/").each_with_object([]) do |template, result| - expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name(template) - next unless expanded_template_name # guard against templates unavailable on FOSS - - event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :repository_source) - - result << ci_template_event(event_name) - end - end - - # rubocop:disable Gitlab/NoCodeCoverageComment - # :nocov: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/299453 - def ci_template_event(event_name) - { 'name' => event_name } - end - # :nocov: - # rubocop:enable Gitlab/NoCodeCoverageComment - - def implicit_auto_devops_event(expanded_template_name) - event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :auto_devops_source) - ci_template_event(event_name) - end end end |