diff options
Diffstat (limited to 'lib')
191 files changed, 2930 insertions, 1256 deletions
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 8208d10c089..0db2321199a 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -37,6 +37,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' @@ -70,6 +71,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, desc: 'Enable or disable Gitlab\'s connection to your Kubernetes cluster' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do diff --git a/lib/api/api.rb b/lib/api/api.rb index b37751e1b47..546d726243e 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -196,6 +196,8 @@ module API mount ::API::ComposerPackages mount ::API::ConanProjectPackages mount ::API::ConanInstancePackages + mount ::API::DebianGroupPackages + mount ::API::DebianProjectPackages mount ::API::MavenPackages mount ::API::NpmPackages mount ::API::GenericPackages @@ -216,6 +218,7 @@ module API mount ::API::ProjectStatistics mount ::API::ProjectTemplates mount ::API::Terraform::State + mount ::API::Terraform::StateVersion mount ::API::ProtectedBranches mount ::API::ProtectedTags mount ::API::Releases @@ -236,6 +239,7 @@ module API mount ::API::Templates mount ::API::Todos mount ::API::Triggers + mount ::API::Unleash mount ::API::UsageData mount ::API::UserCounts mount ::API::Users @@ -245,6 +249,7 @@ module API end mount ::API::Internal::Base + mount ::API::Internal::Lfs mount ::API::Internal::Pages mount ::API::Internal::Kubernetes diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 68497a08fb8..6a86c02bf4a 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -45,7 +45,7 @@ module API def destroy_list(list) destroy_conditionally!(list) do |list| service = ::Boards::Lists::DestroyService.new(board_parent, current_user) - unless service.execute(list) + if service.execute(list).error? render_api_error!({ error: 'List could not be deleted!' }, 400) end end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 08903dce3dc..e293c299d75 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -181,6 +181,7 @@ module API .new(job, declared_params(include_missing: false)) service.execute.then do |result| + header 'X-GitLab-Trace-Update-Interval', result.backoff status result.status end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 20877fb5c5f..3097bcc0ef1 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -62,19 +62,29 @@ module API first_parent: first_parent, order: order) - commit_count = - if all || path || before || after || first_parent - user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent) - else - # Cacheable commit count. - user_project.repository.commit_count_for_ref(ref) - end + serializer = with_stats ? Entities::CommitWithStats : Entities::Commit - paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) + if Feature.enabled?(:api_commits_without_count, user_project) + # This tells kaminari that there is 1 more commit after the one we've + # loaded, meaning there will be a next page, if the currently loaded set + # of commits is equal to the requested page size. + commit_count = offset + commits.size + 1 + paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - serializer = with_stats ? Entities::CommitWithStats : Entities::Commit + present paginate(paginated_commits, exclude_total_headers: true), with: serializer + else + commit_count = + if all || path || before || after || first_parent + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent) + else + # Cacheable commit count. + user_project.repository.commit_count_for_ref(ref) + end - present paginate(paginated_commits), with: serializer + paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) + + present paginate(paginated_commits), with: serializer + end end desc 'Commit multiple file changes as one commit' do diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 31d097c4bea..69e44ffcaf9 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -123,7 +123,7 @@ module API bad_request! end - package_event('push_package') + track_package_event('push_package', :composer) ::Packages::Composer::CreatePackageService .new(authorized_user_project, current_user, declared_params) diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb index 445447cfcd2..9b6867a328b 100644 --- a/lib/api/conan_package_endpoints.rb +++ b/lib/api/conan_package_endpoints.rb @@ -246,7 +246,7 @@ module API delete do authorize!(:destroy_package, project) - package_event('delete_package', category: 'API::ConanPackages') + track_package_event('delete_package', :conan, category: 'API::ConanPackages') package.destroy end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb new file mode 100644 index 00000000000..c56d84ed313 --- /dev/null +++ b/lib/api/debian_group_packages.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + class DebianGroupPackages < Grape::API::Instance + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + not_found! unless ::Feature.enabled?(:debian_packages, user_group) + + authorize_read_package!(user_group) + end + + namespace ':id/-/packages/debian' do + include DebianPackageEndpoints + end + end + end +end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb new file mode 100644 index 00000000000..168b3ca7a4f --- /dev/null +++ b/lib/api/debian_package_endpoints.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +module API + module DebianPackageEndpoints + extend ActiveSupport::Concern + + DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze + COMPONENT_REGEX = %r{[a-z-]+}.freeze + ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze + LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX + DISTRIBUTION_REQUIREMENTS = { + distribution: DISTRIBUTION_REGEX + }.freeze + COMPONENT_ARCHITECTURE_REQUIREMENTS = { + component: COMPONENT_REGEX, + architecture: ARCHITECTURE_REGEX + }.freeze + COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { + component: COMPONENT_REGEX, + letter: LETTER_REGEX, + source_package: PACKAGE_REGEX + }.freeze + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + included do + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + format :txt + + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + before do + require_packages_enabled! + end + + params do + requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex + end + + namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg + desc 'The Release file signature' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Release.gpg' do + not_found! + end + + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release + desc 'The unsigned Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Release' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Release' + end + + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease + desc 'The signed Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'InRelease' do + not_found! + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + end + + namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + desc 'The binary files index' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Packages' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Packages' + end + end + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' + requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + end + + namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name + params do + requires :file_name, type: String, desc: 'The Debian File Name' + end + desc 'The package' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get ':file_name', requirements: FILE_NAME_REQUIREMENTS do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO File' + end + end + end + end +end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb new file mode 100644 index 00000000000..7cd796aac2b --- /dev/null +++ b/lib/api/debian_project_packages.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module API + class DebianProjectPackages < Grape::API::Instance + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + not_found! unless ::Feature.enabled?(:debian_packages, user_project) + + authorize_read_package! + end + + namespace ':id/-/packages/debian' do + include DebianPackageEndpoints + + params do + requires :file_name, type: String, desc: 'The file name' + end + + namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + put do + authorize_upload!(authorized_user_project) + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) + + track_package_event('push_package', :debian) + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) + + forbidden! + end + + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + post 'authorize' do + authorize_workhorse!( + subject: authorized_user_project, + has_length: false, + maximum_size: authorized_user_project.actual_limits.debian_max_file_size + ) + end + end + end + end + end +end diff --git a/lib/api/entities/cluster.rb b/lib/api/entities/cluster.rb index 4cb54e988ce..67459092a33 100644 --- a/lib/api/entities/cluster.rb +++ b/lib/api/entities/cluster.rb @@ -4,7 +4,7 @@ module API module Entities class Cluster < Grape::Entity expose :id, :name, :created_at, :domain - expose :provider_type, :platform_type, :environment_scope, :cluster_type + expose :provider_type, :platform_type, :environment_scope, :cluster_type, :namespace_per_environment expose :user, using: Entities::UserBasic expose :platform_kubernetes, using: Entities::Platform::Kubernetes expose :provider_gcp, using: Entities::Provider::Gcp diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index cff627ab50a..c430b73580b 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -16,6 +16,7 @@ module API expose :project_id expose :location expose :created_at + expose :expiration_policy_started_at, as: :cleanup_policy_started_at expose :tags_count, if: -> (_, options) { options[:tags_count] } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } end diff --git a/lib/api/entities/feature_flag.rb b/lib/api/entities/feature_flag.rb new file mode 100644 index 00000000000..82fdb20af00 --- /dev/null +++ b/lib/api/entities/feature_flag.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + expose :name + expose :description + expose :active + expose :version, if: :feature_flags_new_version_enabled + expose :created_at + expose :updated_at + expose :scopes, using: FeatureFlag::LegacyScope + expose :strategies, using: FeatureFlag::Strategy, if: :feature_flags_new_version_enabled + end + end +end diff --git a/lib/api/entities/feature_flag/detailed_legacy_scope.rb b/lib/api/entities/feature_flag/detailed_legacy_scope.rb new file mode 100644 index 00000000000..47078c1dfde --- /dev/null +++ b/lib/api/entities/feature_flag/detailed_legacy_scope.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class DetailedLegacyScope < LegacyScope + expose :name + end + end + end +end diff --git a/lib/api/entities/feature_flag/legacy_scope.rb b/lib/api/entities/feature_flag/legacy_scope.rb new file mode 100644 index 00000000000..7329f71c599 --- /dev/null +++ b/lib/api/entities/feature_flag/legacy_scope.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class LegacyScope < Grape::Entity + expose :id + expose :active + expose :environment_scope + expose :strategies + expose :created_at + expose :updated_at + end + end + end +end diff --git a/lib/api/entities/feature_flag/scope.rb b/lib/api/entities/feature_flag/scope.rb new file mode 100644 index 00000000000..906fe718257 --- /dev/null +++ b/lib/api/entities/feature_flag/scope.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class Scope < Grape::Entity + expose :id + expose :environment_scope + end + end + end +end diff --git a/lib/api/entities/feature_flag/strategy.rb b/lib/api/entities/feature_flag/strategy.rb new file mode 100644 index 00000000000..32699be0ee3 --- /dev/null +++ b/lib/api/entities/feature_flag/strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class Strategy < Grape::Entity + expose :id + expose :name + expose :parameters + expose :scopes, using: FeatureFlag::Scope + end + end + end +end diff --git a/lib/api/entities/feature_flag/user_list.rb b/lib/api/entities/feature_flag/user_list.rb new file mode 100644 index 00000000000..bc8b12ea22e --- /dev/null +++ b/lib/api/entities/feature_flag/user_list.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class UserList < Grape::Entity + include RequestAwareEntity + + expose :id + expose :iid + expose :project_id + expose :created_at + expose :updated_at + expose :name + expose :user_xids + + expose :path do |list| + project_feature_flags_user_list_path(list.project, list) + end + + expose :edit_path do |list| + edit_project_feature_flags_user_list_path(list.project, list) + end + end + end + end +end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index d903f50befa..b54f0e04a9d 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -7,7 +7,19 @@ module API extend ::API::Entities::EntityHelpers expose :id - expose :name + + expose :name do |package| + if package.conan? + package.conan_recipe + else + package.name + end + end + + expose :conan_package_name, if: ->(package) { package.conan? } do |package| + package.name + end + expose :version expose :package_type diff --git a/lib/api/entities/unleash_feature.rb b/lib/api/entities/unleash_feature.rb new file mode 100644 index 00000000000..8ee87d1fc11 --- /dev/null +++ b/lib/api/entities/unleash_feature.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashFeature < Grape::Entity + expose :name + expose :description, unless: ->(feature) { feature.description.nil? } + expose :active, as: :enabled + expose :strategies do |flag| + flag.strategies.map do |strategy| + if legacy_strategy?(strategy) + UnleashLegacyStrategy.represent(strategy) + elsif gitlab_user_list_strategy?(strategy) + UnleashGitlabUserListStrategy.represent(strategy) + else + UnleashStrategy.represent(strategy) + end + end + end + + private + + def legacy_strategy?(strategy) + !strategy.respond_to?(:name) + end + + def gitlab_user_list_strategy?(strategy) + strategy.name == ::Operations::FeatureFlags::Strategy::STRATEGY_GITLABUSERLIST + end + end + end +end diff --git a/lib/api/entities/unleash_gitlab_user_list_strategy.rb b/lib/api/entities/unleash_gitlab_user_list_strategy.rb new file mode 100644 index 00000000000..5617f8002d9 --- /dev/null +++ b/lib/api/entities/unleash_gitlab_user_list_strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashGitlabUserListStrategy < Grape::Entity + expose :name do |_strategy| + ::Operations::FeatureFlags::Strategy::STRATEGY_USERWITHID + end + expose :parameters do |strategy| + { userIds: strategy.user_list.user_xids } + end + end + end +end diff --git a/lib/api/entities/unleash_legacy_strategy.rb b/lib/api/entities/unleash_legacy_strategy.rb new file mode 100644 index 00000000000..5d5954f8da0 --- /dev/null +++ b/lib/api/entities/unleash_legacy_strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashLegacyStrategy < Grape::Entity + expose :name do |strategy| + strategy['name'] + end + expose :parameters do |strategy| + strategy['parameters'] + end + end + end +end diff --git a/lib/api/entities/unleash_strategy.rb b/lib/api/entities/unleash_strategy.rb new file mode 100644 index 00000000000..7627ce3873c --- /dev/null +++ b/lib/api/entities/unleash_strategy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashStrategy < Grape::Entity + expose :name + expose :parameters + end + end +end diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index c225ade6eb6..ab7bc738ff8 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -8,3 +8,5 @@ module API end end end + +API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin') diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 98b8a40c7c9..a0c29ada950 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -2,6 +2,11 @@ module API class GenericPackages < Grape::API::Instance + GENERIC_PACKAGES_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX, + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + before do require_packages_enabled! authenticate! @@ -17,17 +22,94 @@ module API route_setting :authentication, job_token_allowed: true namespace ':id/packages/generic' do - get 'ping' do - :pong + namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do + desc 'Workhorse authorize generic package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, job_token_allowed: true + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + end + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.generic_packages_max_file_size) + end + + desc 'Upload package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true + + put do + authorize_upload!(project) + bad_request!('File is too large') if max_file_size_exceeded? + + track_event('push_package') + + create_package_file_params = declared_params.merge(build: current_authenticated_job) + ::Packages::Generic::CreatePackageFileService + .new(project, current_user, create_package_file_params) + .execute + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) + + forbidden! + end + + desc 'Download package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + end + + route_setting :authentication, job_token_allowed: true + + get do + authorize_read_package!(project) + + package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) + package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! + + track_event('pull_package') + + present_carrierwave_file!(package_file.file) + end end end end helpers do include ::API::Helpers::PackagesHelpers + include ::API::Helpers::Packages::BasicAuthHelpers def require_generic_packages_available! - not_found! unless Feature.enabled?(:generic_packages, user_project) + not_found! unless Feature.enabled?(:generic_packages, project) + end + + def project + authorized_user_project + end + + def max_file_size_exceeded? + project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size) end end end diff --git a/lib/api/github/entities.rb b/lib/api/github/entities.rb index c28a0b8eb7e..fe228c9a2d2 100644 --- a/lib/api/github/entities.rb +++ b/lib/api/github/entities.rb @@ -119,7 +119,9 @@ module API expose :username, as: :login expose :user_url, as: :url expose :user_url, as: :html_url - expose :avatar_url + expose :avatar_url do |user| + user.avatar_url(only_path: false) + end private diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index ae41d9f13b8..77095ee62e0 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -41,6 +41,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' @@ -74,6 +75,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 25b3059f63b..5b6a3bd36cf 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -4,6 +4,8 @@ module API class GroupContainerRepositories < Grape::API::Instance include PaginationParams + helpers ::API::Helpers::PackagesHelpers + before { authorize_read_group_container_images! } REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( @@ -27,7 +29,7 @@ module API user: current_user, subject: user_group ).execute - track_event('list_repositories') + track_package_event('list_repositories', :container) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 813e41b4d39..efd4b22a591 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -29,7 +29,12 @@ module API # rubocop: disable CodeReuse/ActiveRecord def find_groups(params, parent_id = nil) - find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level) + find_params = params.slice( + :all_available, + :custom_attributes, + :owned, :min_access_level, + :include_parent_descendants + ) find_params[:parent] = if params[:top_level_only] [nil] @@ -309,6 +314,19 @@ module API present_groups params, groups end + desc 'Get a list of descendant groups of this group.' do + success Entities::Group + end + params do + use :group_list_params + use :with_custom_attributes + end + get ":id/descendant_groups" do + finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true) + groups = find_groups(finder_params, params[:id]) + present_groups params, groups + end + desc 'Transfer a project to the group namespace. Available only for admin.' do success Entities::GroupDetail end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1912a06682e..690160cd5ac 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -544,7 +544,6 @@ module API feature_name = "usage_data_#{event_name}" return unless Feature.enabled?(feature_name) - return unless Gitlab::CurrentSettings.usage_ping_enabled? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name) rescue => error diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index dcbf933a4e1..934e18bdd0a 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -158,7 +158,7 @@ module API conan_package_reference: params[:conan_package_reference] ).execute! - package_event('pull_package', category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY + track_package_event('pull_package', :conan, category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY present_carrierwave_file!(package_file.file) end @@ -169,7 +169,7 @@ module API def track_push_package_event if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate - package_event('push_package', category: 'API::ConanPackages') + track_package_event('push_package', :conan, category: 'API::ConanPackages') end end diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 254af7690a2..577ba97d68a 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -10,6 +10,7 @@ module API def redirect_registry_request(forward_to_registry, package_type, options) if forward_to_registry && redirect_registry_request_available? + track_event("#{package_type}_request_forward") redirect(registry_url(package_type, options)) else yield diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 403f5ea3851..e1898d28ef7 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -40,7 +40,7 @@ module API params = { has_length: has_length } params[:maximum_size] = maximum_size unless has_length - ::Packages::PackageFileUploader.workhorse_authorize(params) + ::Packages::PackageFileUploader.workhorse_authorize(**params) end def authorize_upload!(subject = user_project) @@ -48,7 +48,8 @@ module API require_gitlab_workhorse! end - def package_event(event_name, **args) + def track_package_event(event_name, scope, **args) + ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute track_event(event_name, **args) end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index a6ae9a87f98..227aec224e5 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,8 +3,8 @@ module API module Helpers module Pagination - def paginate(relation) - Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) + def paginate(*args) + Gitlab::Pagination::OffsetPagination.new(self).paginate(*args) end end end diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb index a5186cc56ea..f05467ba40b 100644 --- a/lib/api/helpers/presentable.rb +++ b/lib/api/helpers/presentable.rb @@ -23,7 +23,7 @@ module API def initialize(object, options = {}) options = options.opts_hash if options.is_a?(Grape::Entity::Options) - super(object.present(options), options) + super(object.present(options), **options) end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 34a2fb09875..1c85669a626 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -51,9 +51,7 @@ module API job_forbidden!(job, 'Job is not running') unless job.running? end - if Gitlab::Ci::Features.job_heartbeats_runner?(job.project) - job.runner&.heartbeat(get_runner_ip) - end + job.runner&.heartbeat(get_runner_ip) job end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 4bceda51900..4adb27a7414 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -381,6 +381,12 @@ module API type: String, desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' }, + { + required: false, + name: :branches_to_be_notified, + type: String, + desc: 'Branches for which notifications are to be sent' + }, chat_notification_events ].flatten, 'hipchat' => [ diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb index 65aec6ae2e7..451e578fdd6 100644 --- a/lib/api/helpers/settings_helpers.rb +++ b/lib/api/helpers/settings_helpers.rb @@ -12,6 +12,7 @@ module API def self.optional_attributes [*::ApplicationSettingsHelper.visible_attributes, *::ApplicationSettingsHelper.external_authorization_service_attributes, + *::ApplicationSettingsHelper.deprecated_attributes, :performance_bar_allowed_group_id].freeze end end diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb new file mode 100644 index 00000000000..adedc38b847 --- /dev/null +++ b/lib/api/internal/lfs.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module API + module Internal + class Lfs < Grape::API::Instance + use Rack::Sendfile + + before { authenticate_by_gitlab_shell_token! } + + helpers do + def find_lfs_object(lfs_oid) + LfsObject.find_by_oid(lfs_oid) + end + end + + namespace 'internal' do + namespace 'lfs' do + desc 'Get LFS URL for object ID' do + detail 'This feature was introduced in GitLab 13.5.' + end + params do + requires :oid, type: String, desc: 'The object ID to query' + requires :gl_repository, type: String, desc: "Project identifier (e.g. project-1)" + end + get "/" do + lfs_object = find_lfs_object(params[:oid]) + + not_found! unless lfs_object + + _, project, repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) + + not_found! unless repo_type.project? && project + not_found! unless lfs_object.project_allowed_access?(project) + + file = lfs_object.file + + not_found! unless file&.exists? + + content_type 'application/octet-stream' + + if file.file_storage? + sendfile file.path + else + workhorse_headers = Gitlab::Workhorse.send_url(file.url) + header workhorse_headers[0], workhorse_headers[1] + env['api.format'] = :binary + body nil + end + end + end + end + end + end +end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index f7796b1e969..51a87f9433c 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -6,17 +6,22 @@ module API desc 'Validation of .gitlab-ci.yml content' params do requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' end post '/lint' do - error = Gitlab::Ci::YamlProcessor.validation_message(params[:content], - user: current_user) + result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute + error = result.errors.first status 200 - if error.blank? - { status: 'valid', errors: [] } - else - { status: 'invalid', errors: [error] } + response = if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + + response.tap do |response| + response[:merged_yaml] = result.merged_yaml if params[:include_merged_yaml] end end end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index e6d9a9a7c20..d1dd3babb8b 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -107,7 +107,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end end @@ -145,7 +145,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -181,7 +181,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -233,7 +233,7 @@ module API when 'md5' nil else - package_event('push_package') if jar_file?(format) + track_package_event('push_package', :maven) if jar_file?(format) file_params = { file: params[:file], diff --git a/lib/api/npm_packages.rb b/lib/api/npm_packages.rb index fca405b76b7..41238221aad 100644 --- a/lib/api/npm_packages.rb +++ b/lib/api/npm_packages.rb @@ -141,7 +141,7 @@ module API package_file = ::Packages::PackageFileFinder .new(package, params[:file_name]).execute! - package_event('pull_package') + track_package_event('pull_package', package) present_carrierwave_file!(package_file.file) end @@ -157,7 +157,7 @@ module API put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do authorize_create_package!(user_project) - package_event('push_package') + track_package_event('push_package', :npm) created_package = ::Packages::Npm::CreatePackageService .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb index f84a3acbe6d..c0c6efb66b5 100644 --- a/lib/api/nuget_packages.rb +++ b/lib/api/nuget_packages.rb @@ -42,7 +42,7 @@ module API def package_finder(finder_params = {}) ::Packages::Nuget::PackageFinder.new( authorized_user_project, - finder_params.merge(package_name: params[:package_name]) + **finder_params.merge(package_name: params[:package_name]) ) end end @@ -73,7 +73,7 @@ module API get 'index', format: :json do authorize_read_package!(authorized_user_project) - track_event('nuget_service_index') + track_package_event('cli_metadata', :nuget) present ::Packages::Nuget::ServiceIndexPresenter.new(authorized_user_project), with: ::API::Entities::Nuget::ServiceIndex @@ -105,7 +105,7 @@ module API package_file = ::Packages::CreatePackageFileService.new(package, file_params) .execute - package_event('push_package') + track_package_event('push_package', :nuget) ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker @@ -198,7 +198,7 @@ module API not_found!('Package') unless package_file - package_event('pull_package') + track_package_event('pull_package', :nuget) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false present_carrierwave_file!(package_file.file, supports_direct_download: false) @@ -233,7 +233,7 @@ module API .new(authorized_user_project, params[:q], search_options) .execute - package_event('search_package') + track_package_event('search_package', :nuget) present ::Packages::Nuget::SearchResultsPresenter.new(search), with: ::API::Entities::Nuget::SearchResults diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 0e5605984e6..6f189110d76 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -45,6 +45,7 @@ module API optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do @@ -78,6 +79,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 8f2a62bc5a4..173e7799325 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -3,6 +3,7 @@ module API class ProjectContainerRepositories < Grape::API::Instance include PaginationParams + helpers ::API::Helpers::PackagesHelpers REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) @@ -28,7 +29,7 @@ module API user: current_user, subject: user_project ).execute - track_event( 'list_repositories') + track_package_event('list_repositories', :container) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end @@ -43,7 +44,7 @@ module API authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker - track_event('delete_repository') + track_package_event('delete_repository', :container) status :accepted end @@ -60,7 +61,7 @@ module API authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) - track_event('list_tags') + track_package_event('list_tags', :container) present paginate(tags), with: Entities::ContainerRegistry::Tag end @@ -89,7 +90,7 @@ module API declared_params.except(:repository_id).merge(container_expiration_policy: false)) # rubocop:enable CodeReuse/Worker - track_event('delete_tag_bulk') + track_package_event('delete_tag_bulk', :container) status :accepted end @@ -125,7 +126,7 @@ module API .execute(repository) if result[:status] == :success - track_event('delete_tag') + track_package_event('delete_tag', :container) status :ok else diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 377d61689b3..6e4097fd76c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -55,7 +55,7 @@ module API export_strategy = if after_export_params[:url].present? params = after_export_params.slice(:url, :http_method).symbolize_keys - Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(**params) end if export_strategy&.invalid? diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 9f43c3c7993..6ac7f02f305 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -4,8 +4,6 @@ module API class ProjectImport < Grape::API::Instance include PaginationParams - MAXIMUM_FILE_SIZE = 50.megabytes - helpers Helpers::ProjectsHelpers helpers Helpers::FileUploadHelpers helpers Helpers::RateLimiter diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index c07db68f8a8..55cea075243 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -72,7 +72,7 @@ module API package = packages_finder(project).by_file_name_and_sha256(filename, params[:sha256]) package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute - package_event('pull_package') + track_package_event('pull_package', :pypi) present_carrierwave_file!(package_file.file, supports_direct_download: true) end @@ -91,7 +91,7 @@ module API get 'simple/*package_name', format: :txt do authorize_read_package!(authorized_user_project) - package_event('list_package') + track_package_event('list_package', :pypi) packages = find_package_versions presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) @@ -122,7 +122,7 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) - package_event('push_package') + track_package_event('push_package', :pypi) ::Packages::Pypi::CreatePackageService .new(authorized_user_project, current_user, declared_params) diff --git a/lib/api/search.rb b/lib/api/search.rb index b9c6a823f4f..8b6569dd57d 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -62,12 +62,6 @@ module API # Defining this method here as a noop allows us to easily extend it in # EE, without having to modify this file directly. end - - def check_users_search_allowed! - if params[:scope].to_sym == :users && Feature.disabled?(:users_search, default_enabled: true) - render_api_error!({ error: _("Scope not supported with disabled 'users_search' feature!") }, 400) - end - end end resource :search do @@ -85,7 +79,6 @@ module API end get do verify_search_scope!(resource: nil) - check_users_search_allowed! present search, with: entity end @@ -107,7 +100,6 @@ module API end get ':id/(-/)search' do verify_search_scope!(resource: user_group) - check_users_search_allowed! present search(group_id: user_group.id), with: entity end @@ -129,8 +121,6 @@ module API use :pagination end get ':id/(-/)search' do - check_users_search_allowed! - present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 6e5534d0c9a..4056d8602f3 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -29,7 +29,8 @@ module API success Entities::ApplicationSetting end params do - optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :admin_notification_email, type: String, desc: 'Deprecated: Use :abuse_notification_email instead. Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :abuse_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' @@ -194,6 +195,11 @@ module API attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:admin_notification_email) + attrs[:abuse_notification_email] = attrs.delete(:admin_notification_email) + end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb new file mode 100644 index 00000000000..5a4bc620cf6 --- /dev/null +++ b/lib/api/terraform/state_version.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module API + module Terraform + class StateVersion < Grape::API::Instance + default_format :json + + before do + authenticate! + authorize! :read_terraform_state, user_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/terraform/state/:name/versions/:serial' do + params do + requires :name, type: String, desc: 'The name of a Terraform state' + requires :serial, type: Integer, desc: 'The version number of the state' + end + + helpers do + def remote_state_handler + ::Terraform::RemoteStateHandler.new(user_project, current_user, name: params[:name]) + end + + def find_version(serial) + remote_state_handler.find_with_lock do |state| + version = state.versions.find_by_version(serial) + + if version.present? + yield version + else + not_found! + end + end + end + end + + desc 'Get a terraform state version' + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get do + find_version(params[:serial]) do |version| + env['api.format'] = :binary # Bypass json serialization + body version.file.read + status :ok + end + end + + desc 'Delete a terraform state version' + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + delete do + authorize! :admin_terraform_state, user_project + + find_version(params[:serial]) do |version| + version.destroy! + + body false + status :no_content + end + end + end + end + end + end +end diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb new file mode 100644 index 00000000000..8db23c3aaec --- /dev/null +++ b/lib/api/unleash.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module API + class Unleash < Grape::API::Instance + include PaginationParams + + namespace :feature_flags do + resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :project_id, type: String, desc: 'The ID of a project' + optional :instance_id, type: String, desc: 'The Instance ID of Unleash Client' + optional :app_name, type: String, desc: 'The Application Name of Unleash Client' + end + route_param :project_id do + before do + authorize_by_unleash_instance_id! + end + + get do + # not supported yet + status :ok + end + + desc 'Get a list of features (deprecated, v2 client support)' + get 'features' do + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end + + desc 'Get a list of features' + get 'client/features' do + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end + + post 'client/register' do + # not supported yet + status :ok + end + + post 'client/metrics' do + # not supported yet + status :ok + end + end + end + end + + helpers do + def project + @project ||= find_project(params[:project_id]) + end + + def unleash_instance_id + env['HTTP_UNLEASH_INSTANCEID'] || params[:instance_id] + end + + def unleash_app_name + env['HTTP_UNLEASH_APPNAME'] || params[:app_name] + end + + def authorize_by_unleash_instance_id! + unauthorized! unless Operations::FeatureFlagsClient + .find_for_project_and_token(project, unleash_instance_id) + end + + def feature_flags + return [] unless unleash_app_name.present? + + legacy_flags = Operations::FeatureFlagScope.for_unleash_client(project, unleash_app_name) + new_version_flags = Operations::FeatureFlag.for_unleash_client(project, unleash_app_name) + + legacy_flags + new_version_flags + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index 73bb43b88fc..b20ee590124 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -348,7 +348,7 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Get the GPG keys of a specified user. Available only for admins.' do + desc 'Get the GPG keys of a specified user.' do detail 'This feature was added in GitLab 10.0' success Entities::GpgKey end @@ -358,8 +358,6 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/gpg_keys' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) not_found!('User') unless user diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 593f90460ac..08bf395fa98 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -51,7 +51,7 @@ module API def find_project_with_access(params) project = find_project!( - ::Gitlab::Jira::Dvcs.restore_full_path(params.slice(:namespace, :project).symbolize_keys) + ::Gitlab::Jira::Dvcs.restore_full_path(**params.slice(:namespace, :project).symbolize_keys) ) not_found! unless can?(current_user, :download_code, project) project diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index c2266f0bad6..6a45baa60ec 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Artifacts < Files + class Artifacts < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 5e795a449de..9c3b7165de7 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Builds < Files + class Builds < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 0dfe56e214f..514d52d7f65 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Lfs < Files + class Lfs < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index d7aab33d7cb..ae293073ba2 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Pages < Files + class Pages < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index d16ed2facf1..9645a07dfb8 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Registry < Files + class Registry < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb new file mode 100644 index 00000000000..6818d485862 --- /dev/null +++ b/lib/backup/repositories.rb @@ -0,0 +1,236 @@ +# frozen_string_literal: true + +require 'yaml' + +module Backup + class Repositories + attr_reader :progress + + def initialize(progress) + @progress = progress + end + + def dump(max_concurrency:, max_storage_concurrency:) + prepare + + if max_concurrency <= 1 && max_storage_concurrency <= 1 + return dump_consecutive + end + + if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? + raise Error, 'repositories.storages in gitlab.yml is misconfigured' + end + + semaphore = Concurrent::Semaphore.new(max_concurrency) + errors = Queue.new + + threads = Gitlab.config.repositories.storages.keys.map do |storage| + Thread.new do + Rails.application.executor.wrap do + dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + rescue => e + errors << e + end + end + end + + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + + raise errors.pop unless errors.empty? + end + + def restore + Project.find_each(batch_size: 1000) do |project| + restore_repository(project, Gitlab::GlRepository::PROJECT) + restore_repository(project, Gitlab::GlRepository::WIKI) + restore_repository(project, Gitlab::GlRepository::DESIGN) + end + + restore_object_pools + end + + private + + def restore_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).restore(always_create: type.project?) + end + + def backup_repos_path + File.join(Gitlab.config.backup.path, 'repositories') + end + + def prepare + FileUtils.rm_rf(backup_repos_path) + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.mkdir(backup_repos_path, mode: 0700) + end + + def dump_consecutive + Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project| + dump_project(project) + end + end + + def dump_storage(storage, semaphore, max_storage_concurrency:) + errors = Queue.new + queue = InterlockSizedQueue.new(1) + + threads = Array.new(max_storage_concurrency) do + Thread.new do + Rails.application.executor.wrap do + while project = queue.pop + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + semaphore.acquire + end + + begin + dump_project(project) + rescue => e + errors << e + break + ensure + semaphore.release + end + end + end + end + end + + Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project| + break unless errors.empty? + + queue.push(project) + end + + raise errors.pop unless errors.empty? + ensure + queue.close + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + end + + def dump_project(project) + backup_repository(project, Gitlab::GlRepository::PROJECT) + backup_repository(project, Gitlab::GlRepository::WIKI) + backup_repository(project, Gitlab::GlRepository::DESIGN) + end + + def backup_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).backup + end + + def restore_object_pools + PoolRepository.includes(:source_project).find_each do |pool| + progress.puts " - Object pool #{pool.disk_path}..." + + pool.source_project ||= pool.member_projects.first.root_of_fork_network + pool.state = 'none' + pool.save + + pool.schedule + end + end + + class BackupRestore + attr_accessor :progress, :repository, :backup_repos_path + + def initialize(progress, repository, backup_repos_path) + @progress = progress + @repository = repository + @backup_repos_path = backup_repos_path + end + + def backup + progress.puts " * #{display_repo_path} ... " + + if repository.empty? + progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan) + return + end + + FileUtils.mkdir_p(repository_backup_path) + + repository.bundle_to_disk(path_to_bundle) + repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] backing up #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + def restore(always_create: false) + progress.puts " * #{display_repo_path} ... " + + repository.remove rescue nil + + if File.exist?(path_to_bundle) + repository.create_from_bundle(path_to_bundle) + restore_custom_hooks + elsif always_create + repository.create_repository + end + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] restoring #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + private + + def display_repo_path + "#{repository.full_path} (#{repository.disk_path})" + end + + def repository_backup_path + @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) + end + + def path_to_bundle + @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') + end + + def restore_custom_hooks + return unless File.exist?(custom_hooks_tar) + + repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + end + + def custom_hooks_tar + File.join(repository_backup_path, "custom_hooks.tar") + end + end + + class InterlockSizedQueue < SizedQueue + extend ::Gitlab::Utils::Override + + override :pop + def pop(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + + override :push + def push(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb deleted file mode 100644 index eb0b230904e..00000000000 --- a/lib/backup/repository.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: true - -require 'yaml' - -module Backup - class Repository - attr_reader :progress - - def initialize(progress) - @progress = progress - end - - def dump(max_concurrency:, max_storage_concurrency:) - prepare - - if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive - end - - if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? - raise Error, 'repositories.storages in gitlab.yml is misconfigured' - end - - semaphore = Concurrent::Semaphore.new(max_concurrency) - errors = Queue.new - - threads = Gitlab.config.repositories.storages.keys.map do |storage| - Thread.new do - Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue => e - errors << e - end - end - end - - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - - raise errors.pop unless errors.empty? - end - - def backup_project(project) - path_to_project_bundle = path_to_bundle(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .create_bundle(path_to_project_bundle) - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_custom_hooks(project) - FileUtils.mkdir_p(project_backup_path(project)) - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .backup_custom_hooks(custom_hooks_path) - end - - def restore_custom_hooks(project) - return unless Dir.exist?(project_backup_path(project)) - return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) - end - - def restore - Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " - - restore_repo_success = - begin - try_restore_repository(project) - rescue => err - progress.puts "Error: #{err}".color(:red) - false - end - - if restore_repo_success - progress.puts "[DONE]".color(:green) - else - progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) - end - - wiki = ProjectWiki.new(project) - wiki.repository.remove rescue nil - path_to_wiki_bundle = path_to_bundle(wiki) - - if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " - begin - wiki.repository.create_from_bundle(path_to_wiki_bundle) - restore_custom_hooks(wiki) - - progress.puts "[DONE]".color(:green) - rescue => e - progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) - progress.puts "Error #{e}".color(:red) - end - end - end - - restore_object_pools - end - - protected - - def try_restore_repository(project) - path_to_project_bundle = path_to_bundle(project) - project.repository.remove rescue nil - - if File.exist?(path_to_project_bundle) - project.repository.create_from_bundle(path_to_project_bundle) - restore_custom_hooks(project) - else - project.repository.create_repository - end - - true - end - - def path_to_bundle(project) - File.join(backup_repos_path, project.disk_path + '.bundle') - end - - def project_backup_path(project) - File.join(backup_repos_path, project.disk_path) - end - - def custom_hooks_tar(project) - File.join(project_backup_path(project), "custom_hooks.tar") - end - - def backup_repos_path - File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) - end - - private - - def dump_consecutive - Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project| - dump_project(project) - end - end - - def dump_storage(storage, semaphore, max_storage_concurrency:) - errors = Queue.new - queue = InterlockSizedQueue.new(1) - - threads = Array.new(max_storage_concurrency) do - Thread.new do - Rails.application.executor.wrap do - while project = queue.pop - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - semaphore.acquire - end - - begin - dump_project(project) - rescue => e - errors << e - break - ensure - semaphore.release - end - end - end - end - end - - Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project| - break unless errors.empty? - - queue.push(project) - end - - raise errors.pop unless errors.empty? - ensure - queue.close - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - end - - def dump_project(project) - progress.puts " * #{display_repo_path(project)} ... " - - if project.hashed_storage?(:repository) - FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) - else - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - end - - if !empty_repo?(project) - backup_project(project) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan) - end - - wiki = ProjectWiki.new(project) - - if !empty_repo?(wiki) - backup_project(wiki) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan) - end - end - - def progress_warn(project, cmd, output) - progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) - end - - def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_emptiness_caches - project_or_wiki.repository.empty? - end - - def display_repo_path(project) - project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path - end - - def restore_object_pools - PoolRepository.includes(:source_project).find_each do |pool| - progress.puts " - Object pool #{pool.disk_path}..." - - pool.source_project ||= pool.member_projects.first.root_of_fork_network - pool.state = 'none' - pool.save - - pool.schedule - end - end - - class InterlockSizedQueue < SizedQueue - extend ::Gitlab::Utils::Override - - override :pop - def pop(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - - override :push - def push(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - end - end -end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index b6a62bc3f29..9665624f71b 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Uploads < Files + class Uploads < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb index 2ab47c5c6db..1754fec93d4 100644 --- a/lib/banzai/filter/design_reference_filter.rb +++ b/lib/banzai/filter/design_reference_filter.rb @@ -3,8 +3,6 @@ module Banzai module Filter class DesignReferenceFilter < AbstractReferenceFilter - FEATURE_FLAG = :design_management_reference_filter_gfm_pipeline - class Identifier include Comparable attr_reader :issue_iid, :filename @@ -35,14 +33,6 @@ module Banzai self.reference_type = :design - # This filter must be enabled by setting the - # design_management_reference_filter_gfm_pipeline flag - def call - return doc unless enabled? - - super - end - def find_object(project, identifier) records_per_parent[project][identifier] end @@ -112,10 +102,6 @@ module Banzai .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) } end - - def enabled? - Feature.enabled?(FEATURE_FLAG, parent, default_enabled: true) - end end end end diff --git a/lib/banzai/reference_parser/mentioned_group_parser.rb b/lib/banzai/reference_parser/mentioned_group_parser.rb index a0892e15df8..75d05ef59f9 100644 --- a/lib/banzai/reference_parser/mentioned_group_parser.rb +++ b/lib/banzai/reference_parser/mentioned_group_parser.rb @@ -16,7 +16,7 @@ module Banzai end def nodes_visible_to_user(user, nodes) - groups = lazy { grouped_objects_for_nodes(nodes, Group, GROUP_ATTR) } + groups = lazy { grouped_objects_for_nodes(nodes, references_relation, GROUP_ATTR) } nodes.select do |node| node.has_attribute?(GROUP_ATTR) && can_read_group_reference?(node, user, groups) diff --git a/lib/feature.rb b/lib/feature.rb index 71241e98723..1f8c530bee5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -138,7 +138,7 @@ class Feature def register_definitions return unless check_feature_flags_definition? - Feature::Definition.load_all! + Feature::Definition.reload! end def register_hot_reloader diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index ee779a86952..0ba1bdc4799 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -84,17 +84,14 @@ class Feature end def definitions - @definitions ||= {} + # We lazily load all definitions + # The hot reloading might request a feature flag + # before we can properly call `load_all!` + @definitions ||= load_all! end - def load_all! - definitions.clear - - paths.each do |glob_path| - load_all_from_path!(glob_path) - end - - definitions + def reload! + @definitions = load_all! end def valid_usage!(key, type:, default_enabled:) @@ -110,9 +107,7 @@ class Feature def register_hot_reloader! # Reload feature flags on change of this file or any `.yml` file_watcher = Rails.configuration.file_watcher.new(reload_files, reload_directories) do - # We use `Feature::Definition` as on Ruby code-reload - # a new class definition is created - Feature::Definition.load_all! + Feature::Definition.reload! end Rails.application.reloaders << file_watcher @@ -123,6 +118,16 @@ class Feature private + def load_all! + # We currently do not load feature flag definitions + # in production environments + return [] unless Gitlab.dev_or_test_env? + + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + def load_from_file(path) definition = File.read(path) definition = YAML.safe_load(definition) @@ -133,7 +138,7 @@ class Feature raise Feature::InvalidFeatureFlagError, "Invalid definition for `#{path}`: #{e.message}" end - def load_all_from_path!(glob_path) + def load_all_from_path!(definitions, glob_path) Dir.glob(glob_path).each do |path| definition = load_from_file(path) @@ -146,7 +151,7 @@ class Feature end def reload_files - [File.expand_path(__FILE__)] + [] end def reload_directories diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index c06f699ef27..9ec56ee6b52 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -9,12 +9,14 @@ class Feature # optional: defines if a on-disk definition is required for this feature flag type # rollout_issue: defines if `bin/feature-flag` asks for rollout issue # default_enabled: defines a default state of a feature flag when created by `bin/feature-flag` + # ee_only: defines that a feature flag can only be created in a context of EE # example: usage being shown when exception is raised TYPES = { development: { description: 'Short lived, used to enable unfinished code to be deployed', - optional: true, + optional: false, rollout_issue: true, + ee_only: false, default_enabled: false, example: <<-EOS Feature.enabled?(:my_feature_flag, project) @@ -26,6 +28,7 @@ class Feature description: "Long-lived feature flags that control operational aspects of GitLab's behavior", optional: true, rollout_issue: false, + ee_only: false, default_enabled: false, example: <<-EOS Feature.enabled?(:my_ops_flag, type: ops) @@ -36,6 +39,7 @@ class Feature description: 'Permanent feature flags used to temporarily disable licensed features.', optional: true, rollout_issue: false, + ee_only: true, default_enabled: true, example: <<-EOS project.feature_available?(:my_licensed_feature) @@ -44,13 +48,15 @@ class Feature } }.freeze + # The ordering of PARAMS defines an order in YAML + # This is done to ease the file comparison PARAMS = %i[ name - default_enabled - type introduced_by_url rollout_issue_url + type group + default_enabled ].freeze end end diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb deleted file mode 100644 index 3bb839c1114..00000000000 --- a/lib/gitlab/alert_management/alert_params.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module AlertManagement - class AlertParams - MONITORING_TOOLS = { - prometheus: 'Prometheus' - }.freeze - - def self.from_generic_alert(project:, payload:) - parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload, project).with_indifferent_access - annotations = parsed_payload[:annotations] - - { - project_id: project.id, - title: annotations[:title], - description: annotations[:description], - monitoring_tool: annotations[:monitoring_tool], - service: annotations[:service], - hosts: Array(annotations[:hosts]), - payload: payload, - started_at: parsed_payload['startsAt'], - ended_at: parsed_payload['endsAt'], - severity: annotations[:severity], - fingerprint: annotations[:fingerprint], - environment: annotations[:environment] - } - end - - def self.from_prometheus_alert(project:, parsed_alert:) - { - project_id: project.id, - title: parsed_alert.title, - description: parsed_alert.description, - monitoring_tool: MONITORING_TOOLS[:prometheus], - payload: parsed_alert.payload, - started_at: parsed_alert.starts_at, - ended_at: parsed_alert.ends_at, - fingerprint: parsed_alert.gitlab_fingerprint, - environment: parsed_alert.environment, - prometheus_alert: parsed_alert.gitlab_alert - } - end - end - end -end diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb index 382026236e0..e88436d479b 100644 --- a/lib/gitlab/alert_management/alert_status_counts.rb +++ b/lib/gitlab/alert_management/alert_status_counts.rb @@ -30,7 +30,7 @@ module Gitlab end def all - counts.values.sum # rubocop:disable CodeReuse/ActiveRecord + counts.values.sum end private diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 7efdfac75dc..e8e85155bef 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -8,6 +8,8 @@ module Gitlab DEFAULT_TITLE = 'New: Incident' DEFAULT_SEVERITY = 'critical' + attribute :description, paths: 'description' + attribute :ends_at, paths: 'end_time', type: :time attribute :environment_name, paths: 'gitlab_environment_name' attribute :hosts, paths: 'hosts' attribute :monitoring_tool, paths: 'monitoring_tool' @@ -23,3 +25,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic') diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb deleted file mode 100644 index 94b81b7d290..00000000000 --- a/lib/gitlab/alerting/alert.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class Alert - include ActiveModel::Model - include Gitlab::Utils::StrongMemoize - include Presentable - - attr_accessor :project, :payload, :am_alert - - def self.for_alert_management_alert(project:, alert:) - params = if alert.prometheus? - alert.payload - else - Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project) - end - - self.new(project: project, payload: params, am_alert: alert) - end - - def gitlab_alert - strong_memoize(:gitlab_alert) do - parse_gitlab_alert_from_payload - end - end - - def metric_id - strong_memoize(:metric_id) do - payload&.dig('labels', 'gitlab_alert_id') - end - end - - def gitlab_prometheus_alert_id - strong_memoize(:gitlab_prometheus_alert_id) do - payload&.dig('labels', 'gitlab_prometheus_alert_id') - end - end - - def title - strong_memoize(:title) do - gitlab_alert&.title || parse_title_from_payload - end - end - - def description - strong_memoize(:description) do - parse_description_from_payload - end - end - - def environment - strong_memoize(:environment) do - gitlab_alert&.environment || parse_environment_from_payload - end - end - - def annotations - strong_memoize(:annotations) do - parse_annotations_from_payload || [] - end - end - - def starts_at - strong_memoize(:starts_at) do - parse_datetime_from_payload('startsAt') - end - end - - def starts_at_raw - strong_memoize(:starts_at_raw) do - payload&.dig('startsAt') - end - end - - def ends_at - strong_memoize(:ends_at) do - parse_datetime_from_payload('endsAt') - end - end - - def full_query - strong_memoize(:full_query) do - gitlab_alert&.full_query || parse_expr_from_payload - end - end - - def y_label - strong_memoize(:y_label) do - parse_y_label_from_payload || title - end - end - - def alert_markdown - strong_memoize(:alert_markdown) do - parse_alert_markdown_from_payload - end - end - - def status - strong_memoize(:status) do - payload&.dig('status') - end - end - - def firing? - status == 'firing' - end - - def resolved? - status == 'resolved' - end - - def gitlab_managed? - metric_id.present? - end - - def gitlab_fingerprint - Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint) - end - - def valid? - payload.respond_to?(:dig) && project && title && starts_at - end - - def present - super(presenter_class: Projects::Prometheus::AlertPresenter) - end - - private - - def plain_gitlab_fingerprint - if gitlab_managed? - [metric_id, starts_at_raw].join('/') - else # self managed - [starts_at_raw, title, full_query].join('/') - end - end - - def parse_environment_from_payload - environment_name = payload&.dig('labels', 'gitlab_environment_name') - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def parse_gitlab_alert_from_payload - alerts_found = matching_gitlab_alerts - - return if alerts_found.blank? || alerts_found.size > 1 - - alerts_found.first - end - - def matching_gitlab_alerts - return unless metric_id || gitlab_prometheus_alert_id - - Projects::Prometheus::AlertsFinder - .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id) - .execute - end - - def parse_title_from_payload - payload&.dig('annotations', 'title') || - payload&.dig('annotations', 'summary') || - payload&.dig('labels', 'alertname') - end - - def parse_description_from_payload - payload&.dig('annotations', 'description') - end - - def parse_annotations_from_payload - payload&.dig('annotations')&.map do |label, value| - Alerting::AlertAnnotation.new(label: label, value: value) - end - end - - def parse_datetime_from_payload(field) - value = payload&.dig(field) - return unless value - - # value is a rfc3339 timestamp - # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC) - # .utc sets the datetime zone to `UTC` - Time.rfc3339(value).utc - rescue ArgumentError - end - - # Parses `g0.expr` from `generatorURL`. - # - # Example: http://localhost:9090/graph?g0.expr=vector%281%29&g0.tab=1 - def parse_expr_from_payload - url = payload&.dig('generatorURL') - return unless url - - uri = URI(url) - - Rack::Utils.parse_query(uri.query).fetch('g0.expr') - rescue URI::InvalidURIError, KeyError - end - - def parse_alert_markdown_from_payload - payload&.dig('annotations', 'gitlab_incident_markdown') - end - - def parse_y_label_from_payload - payload&.dig('annotations', 'gitlab_y_label') - end - end - end -end diff --git a/lib/gitlab/alerting/alert_annotation.rb b/lib/gitlab/alerting/alert_annotation.rb deleted file mode 100644 index a4b3a97b08c..00000000000 --- a/lib/gitlab/alerting/alert_annotation.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class AlertAnnotation - include ActiveModel::Model - - attr_accessor :label, :value - end - end -end diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb deleted file mode 100644 index 348f851f551..00000000000 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class NotificationPayloadParser - BadPayloadError = Class.new(StandardError) - - DEFAULT_TITLE = 'New: Incident' - DEFAULT_SEVERITY = 'critical' - - def initialize(payload, project) - @payload = payload.to_h.with_indifferent_access - @project = project - end - - def self.call(payload, project) - new(payload, project).call - end - - def call - { - 'annotations' => annotations, - 'startsAt' => starts_at, - 'endsAt' => ends_at - }.compact - end - - private - - attr_reader :payload, :project - - def title - payload[:title].presence || DEFAULT_TITLE - end - - def severity - payload[:severity].presence || DEFAULT_SEVERITY - end - - def fingerprint - Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint]) - end - - def annotations - primary_params - .reverse_merge(flatten_secondary_params) - .transform_values(&:presence) - .compact - end - - def primary_params - { - 'title' => title, - 'description' => payload[:description], - 'monitoring_tool' => payload[:monitoring_tool], - 'service' => payload[:service], - 'hosts' => hosts.presence, - 'severity' => severity, - 'fingerprint' => fingerprint, - 'environment' => environment - } - end - - def hosts - Array(payload[:hosts]).reject(&:blank?) - end - - def current_time - Time.current.change(usec: 0).rfc3339 - end - - def starts_at - Time.parse(payload[:start_time].to_s).rfc3339 - rescue ArgumentError - current_time - end - - def ends_at - Time.parse(payload[:end_time].to_s).rfc3339 - rescue ArgumentError - nil - end - - def environment - environment_name = payload[:gitlab_environment_name] - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def secondary_params - payload.except(:start_time, :end_time) - end - - def flatten_secondary_params - Gitlab::Utils::SafeInlineHash.merge_keys!(secondary_params) - rescue ArgumentError - raise BadPayloadError, 'The payload is too big' - end - end - end -end - -Gitlab::Alerting::NotificationPayloadParser.prepend_if_ee('EE::Gitlab::Alerting::NotificationPayloadParser') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 609eef5e365..001c083c778 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -371,7 +371,7 @@ module Gitlab end def find_build_by_token(token) - ::Ci::Build.running.find_by_token(token) + ::Ci::AuthJobFinder.new(token: token).execute end def user_auth_attempt!(user, success:) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index ccf52bae9a5..3d3f7212053 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -290,7 +290,7 @@ module Gitlab end def api_request? - current_request.path.starts_with?('/api/') + current_request.path.starts_with?(Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/')) end def archive_request? diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 21538000fec..8befade8c3a 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -109,7 +109,7 @@ module Gitlab end def create_commit(snippet) - snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs) + snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs) end # If the user is not allowed to access git or update the snippet diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb index ca64d13b118..bbe2164ae4e 100644 --- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb +++ b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb @@ -13,8 +13,6 @@ module Gitlab end def perform(start_id, stop_id) - return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true) - relation = User .select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio") .where("(COALESCE(bio, '') IS DISTINCT FROM '')") diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb new file mode 100644 index 00000000000..cd305adc7cd --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveDuplicatedCsFindingsWithoutVulnerabilityId + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..3def5eb3369 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + # isolated Banzai::ReferenceParser + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['isolated_mentioned_group'] + # + # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class. + def self.[](name) + const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false) + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb new file mode 100644 index 00000000000..d3d032ba433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Group + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb new file mode 100644 index 00000000000..1d3a3af81a1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Extract possible GFM references from an arbitrary String for further processing. + class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor + REFERABLES = %i(isolated_mentioned_group).freeze + + REFERABLES.each do |type| + define_method("#{type}s") do + @references[type] ||= isolated_references(type) + end + end + + def isolated_references(type) + context = ::Banzai::RenderContext.new(project, current_user) + processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context) + + refs = processor.process(html_documents) + refs[:visible] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index 69ba3f9132b..be9c0ad2b3a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -36,7 +36,8 @@ module Gitlab if extractor extractors[current_user] = extractor else - extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user) + extractor = extractors[current_user] ||= + Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user) extractor.reset_memoized_values end @@ -71,7 +72,7 @@ module Gitlab mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) - mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id)) + mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb new file mode 100644 index 00000000000..5cadfa45b5b --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + module Namespace + # extracted methods for recursive traversing of namespace hierarchy + module RecursiveTraversal + extend ActiveSupport::Concern + + def root_ancestor + return self if persisted? && parent_id.nil? + + strong_memoize(:root_ancestor) do + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors + .reorder(nil) + .find_by(parent_id: nil) + end + end + + # Returns all ancestors, self, and descendants of the current namespace. + def self_and_hierarchy + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .all_objects + end + + # Returns all the ancestors of the current namespaces. + def ancestors + return self.class.none unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: parent_id)) + .base_and_ancestors + end + + # returns all ancestors upto but excluding the given namespace + # when no namespace is given, all ancestors upto the top are returned + def ancestors_upto(top = nil, hierarchy_order: nil) + Gitlab::ObjectHierarchy.new(self.class.where(id: id)) + .ancestors(upto: top, hierarchy_order: hierarchy_order) + end + + def self_and_ancestors(hierarchy_order: nil) + return self.class.where(id: id) unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors(hierarchy_order: hierarchy_order) + end + + # Returns all the descendants of the current namespace. + def descendants + Gitlab::ObjectHierarchy + .new(self.class.where(parent_id: id)) + .base_and_descendants + end + + def self_and_descendants + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_descendants + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb new file mode 100644 index 00000000000..bc04172b9a2 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Group model + class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace + self.store_full_sti_class = false + has_one :saml_provider + + def self.declarative_policy_class + "GroupPolicy" + end + + def max_member_access_for_user(user) + return GroupMember::NO_ACCESS unless user + + return GroupMember::OWNER if user.admin? + + max_member_access = members_with_parents.where(user_id: user) + .reorder(access_level: :desc) + .first + &.access_level + + max_member_access || GroupMember::NO_ACCESS + end + + def members_with_parents + # Avoids an unnecessary SELECT when the group has no parents + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_hierarchy_members = GroupMember.active_without_invites_and_requests + .where(source_id: source_ids) + + GroupMember.from_union([group_hierarchy_members, + members_from_self_and_ancestor_group_shares]) + end + + # rubocop: disable Metrics/AbcSize + def members_from_self_and_ancestor_group_shares + group_group_link_table = GroupGroupLink.arel_table + group_member_table = GroupMember.arel_table + + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids) + cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) + cte_alias = cte.table.alias(GroupGroupLink.table_name) + + # Instead of members.access_level, we need to maximize that access_level at + # the respective group_group_links.group_access. + member_columns = GroupMember.attribute_names.map do |column_name| + if column_name == 'access_level' + smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]], + 'access_level') + else + group_member_table[column_name] + end + end + + GroupMember + .with(cte.to_arel) + .select(*member_columns) + .from([group_member_table, cte.alias_to(group_group_link_table)]) + .where(group_member_table[:requested_at].eq(nil)) + .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) + .where(group_member_table[:source_type].eq('Namespace')) + end + # rubocop: enable Metrics/AbcSize + + def smallest_value_arel(args, column_alias) + Arel::Nodes::As.new( + Arel::Nodes::NamedFunction.new('LEAST', args), + Arel::Nodes::SqlLiteral.new(column_alias)) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb new file mode 100644 index 00000000000..6d7b9a86e69 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Namespace < ApplicationRecord + include ::Gitlab::VisibilityLevel + include ::Gitlab::Utils::StrongMemoize + include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal + + belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + + def visibility_level_field + :visibility_level + end + + def has_parent? + parent_id.present? || parent.present? + end + + # Overridden in EE::Namespace + def feature_available?(_feature) + false + end + end + end + end + end +end + +Namespace.prepend_if_ee('::EE::Namespace') diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 71361b12d07..db7af0088d0 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -20,3 +20,5 @@ module Gitlab end end end + +Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 0373a12ab69..6d152c052dc 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -104,23 +104,24 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] + options = parse_section_options(scanner[4]) section_name = sanitize_section_name(section) - if action == "start" - handle_section_start(scanner, section_name, timestamp) - elsif action == "end" + if action == 'start' + handle_section_start(scanner, section_name, timestamp, options) + elsif action == 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' end end - def handle_section_start(scanner, section, timestamp) + def handle_section_start(scanner, section, timestamp, options) # We make a new line for new section flush_current_line - @state.open_section(section, timestamp) + @state.open_section(section, timestamp, options) # we need to consume match after handling # the open of section, as we want the section @@ -157,6 +158,18 @@ module Gitlab def sanitize_section_name(section) section.to_s.downcase.gsub(/[^a-z0-9]/, '-') end + + def parse_section_options(raw_options) + return unless raw_options + + # We need to remove the square brackets and split + # by comma to get a list of the options + options = raw_options[1...-1].split ',' + + # Now split each option by equals to separate + # each in the format [key, value] + options.to_h { |option| option.split '=' } + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21aa1f84353..b1dee0e1ecc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -32,7 +32,7 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration + :section_header, :section_duration, :section_options def initialize(offset:, style:, sections: []) @offset = offset @@ -68,6 +68,10 @@ module Gitlab @sections << section end + def set_section_options(options) + @section_options = options + end + def set_as_section_header @section_header = true end @@ -90,6 +94,7 @@ module Gitlab result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header result[:section_duration] = @section_duration if @section_duration + result[:section_options] = @section_options if @section_options end end end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 38d36e6950c..b2b6ce649ed 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -26,10 +26,11 @@ module Gitlab Base64.urlsafe_encode64(state.to_json) end - def open_section(section, timestamp) + def open_section(section, timestamp, options) @open_sections[section] = timestamp @current_line.add_section(section) + @current_line.set_section_options(options) @current_line.set_as_section_header end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb index 6af809d46c1..d4ee0978e1b 100644 --- a/lib/gitlab/ci/config/entry/product/matrix.rb +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -46,13 +46,11 @@ module Gitlab end end - # rubocop:disable CodeReuse/ActiveRecord def number_of_generated_jobs value.sum do |config| config.values.reduce(1) { |acc, values| acc * values.size } end end - # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index ac4f70fb69e..2481989060e 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: 2, + minimum: :minimum, too_short: 'requires at least %{count} items' } end @@ -28,6 +28,10 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end + + def minimum + ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index e770187b124..e14d56af978 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.job_heartbeats_runner?(project) - ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end @@ -35,10 +31,6 @@ module Gitlab ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.bulk_insert_on_create?(project) - ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a parituclar project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -54,10 +46,6 @@ module Gitlab Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.coverage_report_view?(project) - ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) - end - def self.child_of_child_pipeline_enabled?(project) ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) end @@ -72,7 +60,11 @@ module Gitlab end def self.new_artifact_file_reader_enabled?(project) - ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) + ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: true) + end + + def self.one_dimensional_matrix_enabled? + ::Feature.enabled?(:one_dimensional_matrix, default_enabled: false) end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d1882059dd8..06096a33f27 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -16,7 +16,7 @@ module Gitlab ) do include Gitlab::Utils::StrongMemoize - def initialize(**params) + def initialize(params = {}) params.each do |key, value| self[key] = value end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 34649fe16f3..81ef3bb074d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,7 @@ module Gitlab include Chain::Helpers def perform! - BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + BulkInsertableAssociations.with_bulk_insert do pipeline.save! end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb new file mode 100644 index 00000000000..95d7719e9cb --- /dev/null +++ b/lib/gitlab/ci/runner/backoff.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + ## + # Runner Backoff class is an implementation of an exponential backoff + # used when a runner communicates with GitLab. We typically use it when a + # runner retries sending a build status after we created a build pending + # state. + # + # Backoff is calculated based on the backoff slot which is always a power + # of 2: + # + # 0s - 3s duration -> 1 second backoff + # 4s - 7s duration -> 2 seconds backoff + # 8s - 15s duration -> 4 seconds backoff + # 16s - 31s duration -> 8 seconds backoff + # 32s - 63s duration -> 16 seconds backoff + # 64s - 127s duration -> 32 seconds backoff + # 127s - 256s+ duration -> 64 seconds backoff + # + # It means that first 15 requests made by a runner will need to respect + # following backoffs: + # + # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff) + # 1s -> 1 second backoff + # 2s -> 1 second backoff + # 3s -> 1 seconds backoff + # (slot 1 - 2^1 backoff) + # 4s -> 2 seconds backoff + # 6s -> 2 seconds backoff + # (slot 2 - 2^2 backoff) + # 8s -> 4 seconds backoff + # 12s -> 4 seconds backoff + # (slot 3 - 2^3 backoff) + # 16s -> 8 seconds backoff + # 24s -> 8 seconds backoff + # (slot 4 - 2^4 backoff) + # 32s -> 16 seconds backoff + # 48s -> 16 seconds backoff + # (slot 5 - 2^5 backoff) + # 64s -> 32 seconds backoff + # 96s -> 32 seconds backoff + # (slot 6 - 2^6 backoff) + # 128s -> 64 seconds backoff + # + # There is a cap on the backoff - it will never exceed 64 seconds. + # + class Backoff + def initialize(started) + @started = started + + if duration < 0 + raise ArgumentError, 'backoff duration negative' + end + end + + def duration + (Time.current - @started).ceil + end + + def slot + return 0 if duration < 2 + + Math.log(duration, 2).floor - 1 + end + + def to_seconds + 2**[slot, 6].min + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 829fd7a722f..8b921305c11 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index da474f8ac88..317e8bfab0e 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -10,6 +10,7 @@ .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + dependencies: [] script: - ecs update-task-definition diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 348e5472cb4..6fd32b3f1a0 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -16,6 +16,7 @@ module Gitlab ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) + LockedError = Class.new(StandardError) attr_reader :job @@ -130,6 +131,12 @@ module Gitlab end end + def lock(&block) + in_write_lock(&block) + rescue FailedToObtainLockError + raise LockedError, "build trace `#{job.id}` is locked" + end + private def read_stream diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb new file mode 100644 index 00000000000..b01136a6d24 --- /dev/null +++ b/lib/gitlab/ci/trace/checksum.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Checksum class is responsible for calculating a CRC32 checksum + # of an entire build trace using partial build trace chunks stored in a + # database. + # + # CRC32 checksum can be easily calculated by combining partial checksums + # in a right order. + # + # Then we compare CRC32 checksum provided by a GitLab Runner and expect + # it to be the same as the CRC32 checksum derived from partial chunks. + # + class Checksum + include Gitlab::Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def valid? + return false unless state_crc32.present? + + state_crc32 == chunks_crc32 + end + + def state_crc32 + strong_memoize(:crc32) { build.pending_state&.crc32 } + end + + def chunks_crc32 + trace_chunks.reduce(0) do |crc32, chunk| + Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk)) + end + end + + def last_chunk + strong_memoize(:last_chunk) { trace_chunks.max } + end + + ## + # Trace chunks will be persisted in a database if an object store is + # not configured - in that case we do not want to load entire raw data + # of all the chunks into memory. + # + # We ignore `raw_data` attribute instead, and rely on internal build + # trace chunk database adapter to handle + # `ActiveModel::MissingAttributeError` exception. + # + # Alternative solution would be separating chunk data from chunk + # metadata on the database level too. + # + def trace_chunks + strong_memoize(:trace_chunks) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end + end + + private + + def chunk_size(chunk) + if chunk == last_chunk + chunk.size + else + ::Ci::BuildTraceChunk::CHUNK_SIZE + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 82a7d5fb83c..51372871f39 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -7,7 +7,8 @@ module Gitlab extend Gitlab::Utils::StrongMemoize OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict].freeze + :accepted, :finalized, :discarded, :conflict, :locked, + :invalid].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) @@ -18,7 +19,7 @@ module Gitlab end def increment_trace_bytes(size) - self.class.trace_bytes.increment(by: size.to_i) + self.class.trace_bytes.increment({}, size.to_i) end def self.trace_operations diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index a072036daa8..84a9280e507 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -36,9 +36,9 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource.symbolize_keys) + self.new(**resource.symbolize_keys) when ::Ci::HasVariable - self.new(resource.to_runner_variable) + self.new(**resource.to_runner_variable) when self resource.dup else diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 68f61e52df7..6c771b220ad 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -95,6 +95,10 @@ module Gitlab }.compact }.compact end + def merged_yaml + @ci_config&.to_hash&.to_yaml + end + private def variables diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index 14eac474e27..a6638b2cbc8 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -19,6 +19,11 @@ module Gitlab def run! log_info("Looking for orphan LFS files for project #{project.name_with_namespace}") + if project.lfs_objects.empty? + log_info("Project #{project.name_with_namespace} is linked to 0 LFS objects. Nothing to do") + return + end + remove_orphan_references end diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index 909d0536b5f..7d36f2f12cf 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -13,7 +13,6 @@ module Gitlab end def full_json_path_for(path) - return unless Feature.enabled?(:code_navigation, project, default_enabled: true) return unless build raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 7c5ffaa7621..f76c98f7cbf 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -79,7 +79,7 @@ module Gitlab end def fabricate(entry_class, value = nil) - entry_class.new(value, @metadata) do |node| + entry_class.new(value, **@metadata) do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] node.default = @attributes[:default] diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index 315f1947e2c..ee28891a174 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -19,7 +19,7 @@ module Gitlab entry = self.class.entry_class(strategy) - @subject = entry.new(config, metadata, &blk) + @subject = entry.new(config, **metadata, &blk) super(@subject) end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 53406af2c4e..047600af267 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -21,11 +21,13 @@ module Gitlab def resolve(user, commit_message, files) msg = commit_message || default_commit_message resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) - args = { + + resolver.resolve_conflicts( + @source_repo, + resolution, source_branch: merge_request.source_branch, target_branch: merge_request.target_branch - } - resolver.resolve_conflicts(@source_repo, resolution, args) + ) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 954934518d7..7b01db125a9 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -10,7 +10,7 @@ module Gitlab MAX_LINE_LENGTH = 72 MAX_CHANGED_FILES_IN_COMMIT = 3 MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze + SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' WIP_PREFIX = 'WIP: ' PROBLEMS = { @@ -118,7 +118,7 @@ module Gitlab next unless line_too_long?(line) - url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord + url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but # only if the line _without_ the URL does not exceed this limit. diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 3626ec5bf5b..d01455c9ec4 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -214,6 +214,12 @@ module Gitlab title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') end + def draft_mr? + return false unless gitlab_helper + + DRAFT_REGEX.match?(gitlab_helper.mr_json['title']) + end + def security_mr? return false unless gitlab_helper diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index a6866868e6c..e67e4a45bfe 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -146,13 +146,19 @@ module Gitlab %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) end + hungry_reviewers = reviewers.select { |member| member.hungry } + hungry_traintainers = traintainers.select { |member| member.hungry } # TODO: take CODEOWNERS into account? # https://gitlab.com/gitlab-org/gitlab/issues/26723 - # Make traintainers have triple the chance to be picked as a reviewer random = new_random(mr_source_branch) - reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment) + + # Make hungry traintainers have 4x the chance to be picked as a reviewer + # Make traintainers have 3x the chance to be picked as a reviewer + # Make hungry reviewers have 2x the chance to be picked as a reviewer + weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers + reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) Spin.new(category, reviewer, maintainer, false, timezone_experiment) diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index ebd96be40d7..4481977db15 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,7 +3,7 @@ module Gitlab module Danger class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) @@ -14,6 +14,7 @@ module Gitlab @role = options['role'] @projects = options['projects'] @available = options['available'] + @hungry = options['hungry'] @tz_offset_hours = options['tz_offset_hours'] end @@ -31,10 +32,8 @@ module Gitlab projects&.has_key?(name) end - # Traintainers also count as reviewers def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) || - traintainer?(project, category, labels) + has_capability?(project, category, :reviewer, labels) end def traintainer?(project, category, labels) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index accc6330253..d9c4b1cb280 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -92,10 +92,6 @@ module Gitlab @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end - def self.postgresql_9_or_less? - version.to_f < 10 - end - def self.postgresql_minimum_supported_version? version.to_f >= MINIMUM_POSTGRES_VERSION end @@ -127,28 +123,6 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - # map some of the function names that changed between PostgreSQL 9 and 10 - # https://wiki.postgresql.org/wiki/New_in_postgres_10 - def self.pg_wal_lsn_diff - Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff' - end - - def self.pg_current_wal_insert_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn' - end - - def self.pg_last_wal_receive_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn' - end - - def self.pg_last_wal_replay_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn' - end - - def self.pg_last_xact_replay_timestamp - 'pg_last_xact_replay_timestamp' - end - def self.nulls_last_order(field, direction = 'ASC') Arel.sql("#{field} #{direction} NULLS LAST") end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 1762b81b7d8..0de67ed8cf0 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -8,15 +8,20 @@ # In order to not use a possible complex time consuming query when calculating min and max for batch_distinct_count # the start and finish can be sent specifically # +# Grouped relations can be used as well. However, the preferred batch count should be around 10K because group by count is more expensive. +# # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 # # Examples: # extend ::Gitlab::Database::BatchCount # batch_count(User.active) # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +# batch_count(Namespace.group(:type)) # batch_distinct_count(::Project, :creator_id) # batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) +# batch_sum(Issue.group(:state_id), :weight)) module Gitlab module Database module BatchCount @@ -77,12 +82,12 @@ module Gitlab raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 return FALLBACK if unwanted_configuration?(finish, batch_size, start) - counter = 0 + results = nil batch_start = start while batch_start <= finish begin - counter += batch_fetch(batch_start, batch_start + batch_size, mode) + results = merge_results(results, batch_fetch(batch_start, batch_start + batch_size, mode)) batch_start += batch_size rescue ActiveRecord::QueryCanceled # retry with a safe batch size & warmer cache @@ -95,7 +100,17 @@ module Gitlab sleep(SLEEP_TIME_IN_SECONDS) end - counter + results + end + + def merge_results(results, object) + return object unless results + + if object.is_a?(Hash) + results.merge!(object) { |_, a, b| a + b } + else + results + object + end end def batch_fetch(start, finish, mode) @@ -118,11 +133,11 @@ module Gitlab end def actual_start(start) - start || @relation.minimum(@column) || 0 + start || @relation.unscope(:group, :having).minimum(@column) || 0 end def actual_finish(finish) - finish || @relation.maximum(@column) || 0 + finish || @relation.unscope(:group, :having).maximum(@column) || 0 end def check_mode!(mode) diff --git a/lib/gitlab/database/concurrent_reindex.rb b/lib/gitlab/database/concurrent_reindex.rb deleted file mode 100644 index 485ab35e55d..00000000000 --- a/lib/gitlab/database/concurrent_reindex.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - include MigrationHelpers - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - - attr_reader :index_name, :logger - - def initialize(index_name, logger:) - @index_name = index_name - @logger = logger - end - - def execute - raise ReindexError, "index #{index_name} does not exist" unless index_exists? - - raise ReindexError, 'UNIQUE indexes are currently not supported' if index_unique? - - logger.debug("dropping dangling index from previous run: #{replacement_index_name}") - remove_replacement_index - - begin - create_replacement_index - - unless replacement_index_valid? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index_name}: #{message}" - end - - swap_replacement_index - rescue Gitlab::Database::WithLockRetries::AttemptsExhaustedError => e - logger.error('failed to obtain the required database locks to swap the indexes, cleaning up') - raise ReindexError, e.message - rescue ActiveRecord::ActiveRecordError, PG::Error => e - logger.error("database error while attempting reindex of #{index_name}: #{e.message}") - raise ReindexError, e.message - ensure - logger.info("dropping unneeded replacement index: #{replacement_index_name}") - remove_replacement_index - end - end - - private - - def connection - @connection ||= ActiveRecord::Base.connection - end - - def replacement_index_name - @replacement_index_name ||= constrained_index_name(TEMPORARY_INDEX_PREFIX) - end - - def index - strong_memoize(:index) do - find_index(index_name) - end - end - - def index_exists? - !index.nil? - end - - def index_unique? - index.indisunique - end - - def constrained_index_name(prefix) - "#{prefix}#{index_name}".slice(0, PG_IDENTIFIER_LENGTH) - end - - def create_replacement_index - create_replacement_index_statement = index.indexdef - .sub(/CREATE INDEX/, 'CREATE INDEX CONCURRENTLY') - .sub(/#{index_name}/, replacement_index_name) - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - disable_statement_timeout do - connection.execute(create_replacement_index_statement) - end - end - - def replacement_index_valid? - find_index(replacement_index_name).indisvalid - end - - def find_index(index_name) - record = connection.select_one(<<~SQL) - SELECT - pg_index.indisunique, - pg_index.indisvalid, - pg_indexes.indexdef - FROM pg_index - INNER JOIN pg_class ON pg_class.oid = pg_index.indexrelid - INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - INNER JOIN pg_indexes ON pg_class.relname = pg_indexes.indexname - WHERE pg_namespace.nspname = 'public' - AND pg_class.relname = #{connection.quote(index_name)} - SQL - - OpenStruct.new(record) if record - end - - def swap_replacement_index - replaced_index_name = constrained_index_name(REPLACED_INDEX_PREFIX) - - logger.info("swapping replacement index #{replacement_index_name} with #{index_name}") - - with_lock_retries do - rename_index(index_name, replaced_index_name) - rename_index(replacement_index_name, index_name) - rename_index(replaced_index_name, replacement_index_name) - end - end - - def rename_index(old_index_name, new_index_name) - connection.execute("ALTER INDEX #{old_index_name} RENAME TO #{new_index_name}") - end - - def remove_replacement_index - disable_statement_timeout do - connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{replacement_index_name}") - end - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - - Gitlab::Database::WithLockRetries.new(arguments).run(raise_on_exhaustion: true, &block) - end - end - end -end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index e226ed7613a..89190320cf9 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -74,8 +74,9 @@ module Gitlab def get_statistics(table_names, check_statistics: true) time = 6.hours.ago - query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") .where(relname: table_names) + .where('schemaname = current_schema()') .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') if check_statistics diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 723f0f6a308..4e2e1eaf21c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -176,7 +176,7 @@ module Gitlab name: name.presence || concurrent_foreign_key_name(source, column) } - if foreign_key_exists?(source, target, options) + if foreign_key_exists?(source, target, **options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ @@ -330,13 +330,13 @@ module Gitlab # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` # * +logger+ - [Gitlab::JsonLogger] # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` - def with_lock_retries(**args, &block) + def with_lock_retries(*args, **kwargs, &block) merged_args = { klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }.merge(args) + }.merge(kwargs) - Gitlab::Database::WithLockRetries.new(merged_args).run(&block) + Gitlab::Database::WithLockRetries.new(**merged_args).run(&block) end def true_value @@ -882,7 +882,7 @@ module Gitlab # column. opclasses[new] = opclasses.delete(old) if opclasses[old] - options[:opclasses] = opclasses + options[:opclass] = opclasses end add_concurrent_index(table, new_columns, options) @@ -994,10 +994,10 @@ into similar problems in the future (e.g. when new tables are created). def postgres_exists_by_name?(table, name) index_sql = <<~SQL SELECT COUNT(*) - FROM pg_index - JOIN pg_class i ON (indexrelid=i.oid) - JOIN pg_class t ON (indrelid=t.oid) - WHERE i.relname = '#{name}' AND t.relname = '#{table}' + FROM pg_catalog.pg_indexes + WHERE schemaname = #{connection.quote(current_schema)} + AND tablename = #{connection.quote(table)} + AND indexname = #{connection.quote(name)} SQL connection.select_value(index_sql).to_i > 0 @@ -1053,11 +1053,15 @@ into similar problems in the future (e.g. when new tables are created). # the table name in addition to using the constraint_name check_sql = <<~SQL SELECT COUNT(*) - FROM pg_constraint - JOIN pg_class ON pg_constraint.conrelid = pg_class.oid - WHERE pg_constraint.contype = 'c' - AND pg_constraint.conname = '#{constraint_name}' - AND pg_class.relname = '#{table}' + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table)} SQL connection.select_value(check_sql) > 0 @@ -1284,8 +1288,9 @@ into similar problems in the future (e.g. when new tables are created). check_sql = <<~SQL SELECT c.is_nullable FROM information_schema.columns c - WHERE c.table_name = '#{table}' - AND c.column_name = '#{column}' + WHERE c.table_schema = #{connection.quote(current_schema)} + AND c.table_name = #{connection.quote(table)} + AND c.column_name = #{connection.quote(column)} SQL connection.select_value(check_sql) == 'YES' diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 4c1b13fe3b5..547e0b9b957 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -72,10 +72,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::AppLogger - }).run(&block) + ).run(&block) end def connection diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index f9ad1e60776..17a42d997e6 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -11,8 +11,6 @@ module Gitlab PAUSE_SECONDS = 0.25 def perform(start_id, stop_id, source_table, partitioned_table, source_column) - return unless Feature.enabled?(:backfill_partitioned_audit_events, default_enabled: true) - if transaction_open? raise "Aborting job to backfill partitioned #{source_table} table! Do not run this job in a transaction block!" end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb new file mode 100644 index 00000000000..2a9f23f0098 --- /dev/null +++ b/lib/gitlab/database/postgres_index.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresIndex < ActiveRecord::Base + self.table_name = 'postgres_indexes' + self.primary_key = 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + # A 'regular' index is a non-unique index, + # that does not serve an exclusion constraint and + # is defined on a table that is not partitioned. + scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)} + + scope :random_few, ->(how_many) do + limit(how_many).order(Arel.sql('RANDOM()')) + end + + scope :not_match, ->(regex) { where("name !~ ?", regex)} + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb new file mode 100644 index 00000000000..baffe28d9ed --- /dev/null +++ b/lib/gitlab/database/reindexing.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + def self.perform(index_selector) + Array.wrap(index_selector).each do |index| + ReindexAction.keep_track_of(index) do + ConcurrentReindex.new(index).perform + end + end + end + + def self.candidate_indexes + Gitlab::Database::PostgresIndex + .regular + .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") + .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + end + end + end +end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb new file mode 100644 index 00000000000..89fab4a183c --- /dev/null +++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ConcurrentReindex + include Gitlab::Utils::StrongMemoize + include MigrationHelpers + + ReindexError = Class.new(StandardError) + + PG_IDENTIFIER_LENGTH = 63 + TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' + REPLACED_INDEX_PREFIX = 'old_reindex_' + + attr_reader :index, :logger + + def initialize(index, logger: Gitlab::AppLogger) + @index = index + @logger = logger + end + + def perform + raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? + raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? + raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) + + logger.info "Starting reindex of #{index}" + + with_rebuilt_index do |replacement_index| + swap_index(replacement_index) + end + end + + private + + def with_rebuilt_index + if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") + remove_index(index.schema, replacement_index_name) + end + + create_replacement_index_statement = index.definition + .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") + + logger.info("creating replacement index #{replacement_index_name}") + logger.debug("replacement index definition: #{create_replacement_index_statement}") + + disable_statement_timeout do + connection.execute(create_replacement_index_statement) + end + + replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + + unless replacement_index.valid_index? + message = 'replacement index was created as INVALID' + logger.error("#{message}, cleaning up") + raise ReindexError, "failed to reindex #{index}: #{message}" + end + + yield replacement_index + ensure + begin + remove_index(index.schema, replacement_index_name) + rescue => e + logger.error(e) + end + end + + def swap_index(replacement_index) + logger.info("swapping replacement index #{replacement_index} with #{index}") + + with_lock_retries do + rename_index(index.schema, index.name, replaced_index_name) + rename_index(replacement_index.schema, replacement_index.name, index.name) + rename_index(index.schema, replaced_index_name, replacement_index.name) + end + end + + def rename_index(schema, old_index_name, new_index_name) + connection.execute(<<~SQL) + ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} + RENAME TO #{quote_table_name(new_index_name)} + SQL + end + + def remove_index(schema, name) + logger.info("Removing index #{schema}.#{name}") + + disable_statement_timeout do + connection.execute(<<~SQL) + DROP INDEX CONCURRENTLY + IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} + SQL + end + end + + def replacement_index_name + @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" + end + + def replaced_index_name + @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + + Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) + end + + delegate :execute, :quote_table_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb new file mode 100644 index 00000000000..0928ef90e5d --- /dev/null +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ReindexAction < ActiveRecord::Base + self.table_name = 'postgres_reindex_actions' + + enum state: { started: 0, finished: 1, failed: 2 } + + def self.keep_track_of(index, &block) + action = create!( + index_identifier: index.identifier, + action_start: Time.zone.now, + ondisk_size_bytes_start: index.ondisk_size_bytes + ) + + yield + + action.state = :finished + rescue + action.state = :failed + raise + ensure + index.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + action.action_end = Time.zone.now + action.ondisk_size_bytes_end = index.ondisk_size_bytes + + action.save! + end + end + end + end +end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index dda4d8eecdb..3d929c62933 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -32,11 +32,14 @@ module Gitlab def trigger_exists?(table_name, name) connection.select_value(<<~SQL) SELECT 1 - FROM pg_trigger - INNER JOIN pg_class - ON pg_trigger.tgrelid = pg_class.oid - WHERE pg_class.relname = '#{table_name}' - AND pg_trigger.tgname = '#{name}' + FROM pg_catalog.pg_trigger trgr + INNER JOIN pg_catalog.pg_class rel + ON trgr.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = rel.relnamespace + WHERE nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table_name)} + AND trgr.tgname = #{connection.quote(name)} SQL end @@ -68,10 +71,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }).run(&block) + ).run(&block) end def assert_not_in_transaction_block(scope:) diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 2633c29438a..ff78fd0218c 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -6,6 +6,11 @@ module Gitlab EMPTY_STRING = Arel.sql("''").freeze EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze DEFAULT_MULTIPLIER = 1 + DISPLAY_NAME = self.name.underscore.freeze + + # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity. + # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE... + SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. # @@ -74,6 +79,10 @@ module Gitlab end end + def self.order_by_similarity?(arel_query) + arel_query.to_sql.include?(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION) + end + # (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric)) def self.rule_to_arel(search, rule) Arel::Nodes::Grouping.new( @@ -91,7 +100,7 @@ module Gitlab # SIMILARITY(COALESCE(column, ''), 'search_string') def self.similarity_function_call(search, column) - Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)]) + Arel::Nodes::NamedFunction.new(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION, [column, Arel.sql(search)]) end # CAST(multiplier AS numeric) diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index a9c86e4e267..3fb52d786ad 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -95,7 +95,7 @@ module Gitlab run_block_with_transaction rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? - disable_idle_in_transaction_timeout + disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? wait_until_next_retry reset_db_settings @@ -149,7 +149,7 @@ module Gitlab log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) log(message: "Executing the migration without lock timeout", current_iteration: current_iteration) - execute("SET LOCAL lock_timeout TO '0'") + disable_lock_timeout if ActiveRecord::Base.connection.transaction_open? run_block @@ -184,6 +184,10 @@ module Gitlab execute("SET LOCAL idle_in_transaction_session_timeout TO '0'") end + def disable_lock_timeout + execute("SET LOCAL lock_timeout TO '0'") + end + def reset_db_settings execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout') end diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml new file mode 100644 index 00000000000..1d341e6520e --- /dev/null +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -0,0 +1,43 @@ +# This file exists to lock the attributes of Design Management models +# that get copied in `DesignManagement::CopyDesignCollection::CopyService` +# to specific schemas. +# +# This allows us to perform sanity checks and alert when there are changes +# to the schema by running expectations against the lists in this file +# and the actual schema of the models in `copy_designs_service_spec.rb`. +# +# If you are here because you received a failed test in +# `copy_designs_service_spec.rb`, you need to decide how to handle the +# changes and whether the new attribute(s) should be included in the copy +# or ignored. + +# COPY. +# Add attributes that should be copied to the `{model}_attributes` lists: +design_attributes: + - filename + - relative_position + +version_attributes: + - author_id + - created_at + +action_attributes: # (None) + +# IGNORE. +# Add attributes that should not be copied to the `ignore_{model}_attributes` lists: +ignore_design_attributes: + - id + - issue_id + - project_id + +ignore_version_attributes: + - id + - issue_id + - sha + +ignore_action_attributes: + - id + - design_id + - event + - image_v432x230 + - version_id diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d54e1aad19a..341572f9c94 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -11,7 +11,7 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, - diff_options: diff_options, + diff_options: merged_diff_options(diff_options), diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end @@ -64,6 +64,11 @@ module Gitlab diff_stats_cache.read || super end end + + def merged_diff_options(diff_options) + max_diff_options = ::Commit.max_diff_options(project: @merge_request_diff.project) + diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options + end end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 0eb22e6b3cb..e873e9c17d5 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -20,10 +20,23 @@ module Gitlab # - Assigns DiffFile#highlighted_diff_lines for cached files # def decorate(diff_file) - if content = read_file(diff_file) - diff_file.highlighted_diff_lines = content.map do |line| - Gitlab::Diff::Line.safe_init_from_hash(line) - end + content = read_file(diff_file) + + return [] unless content + + if content.empty? && recache_due_to_size?(diff_file) + # If the file is missing from the cache and there's reason to believe + # it is uncached due to a size issue around changing the values for + # max patch size, manually populate the hash and then set the value. + # + new_cache_content = {} + new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + + write_to_redis_hash(new_cache_content) + + set_highlighted_diff_lines(diff_file, read_file(diff_file)) + else + set_highlighted_diff_lines(diff_file, content) end end @@ -58,6 +71,28 @@ module Gitlab private + def set_highlighted_diff_lines(diff_file, content) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.safe_init_from_hash(line) + end + end + + def recache_due_to_size?(diff_file) + diff_file_class = diff_file.diff.class + + current_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes + default_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes(diff_file_class::DEFAULT_MAX_PATCH_BYTES) + + # If the diff is >= than the default limit, but less than the current + # limit, it is likely uncached due to having hit the default limit, + # making it eligible for recalculating. + # + diff_file.diff.diff_bytesize.between?( + default_patch_safe_limit_bytes, + current_patch_safe_limit_bytes + ) + end + def cacheable_files strong_memoize(:cacheable_files) do diff_files.select { |file| cacheable?(file) && read_file(file).nil? } diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 10762d83588..da5b0afad38 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -35,7 +35,7 @@ module Gitlab lease.obtain(1 + retries) - yield(lease.retried?) + yield(lease.retried?, lease) ensure lease&.cancel end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index dca60c93fb2..72d3da6f0f2 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -62,6 +62,12 @@ module Gitlab }, invite_email: { tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' + }, + invitation_reminders: { + tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders' + }, + group_only_trials: { + tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials' } }.freeze @@ -91,10 +97,17 @@ module Gitlab } end + def push_frontend_experiment(experiment_key) + var_name = experiment_key.to_s.camelize(:lower) + enabled = experiment_enabled?(experiment_key) + + gon.push({ experiments: { var_name => enabled } }, true) + end + def experiment_enabled?(experiment_key) return false if dnt_enabled? - return true if Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) + return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index) return true if forced_enabled?(experiment_key) false @@ -102,7 +115,7 @@ module Gitlab def track_experiment_event(experiment_key, action, value = nil) track_experiment_event_for(experiment_key, action, value) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), tracking_data) + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) end end @@ -183,9 +196,14 @@ module Gitlab experiment.enabled? && experiment.enabled_for_environment? end - def enabled_for_user?(experiment_key, experimentation_subject_index) + def enabled_for_attribute?(experiment_key, attribute) + index = Digest::SHA1.hexdigest(attribute).hex % 100 + enabled_for_value?(experiment_key, index) + end + + def enabled_for_value?(experiment_key, experimentation_subject_index) enabled?(experiment_key) && - experiment(experiment_key).enabled_for_experimentation_subject?(experimentation_subject_index) + experiment(experiment_key).enabled_for_index?(experimentation_subject_index) end end @@ -200,10 +218,10 @@ module Gitlab environment end - def enabled_for_experimentation_subject?(experimentation_subject_index) - return false if experimentation_subject_index.blank? + def enabled_for_index?(index) + return false if index.blank? - experimentation_subject_index <= experiment_percentage + index <= experiment_percentage end private diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 09a49b6c1ca..78c47023c08 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -120,8 +120,8 @@ module Gitlab # default. # # Patches surpassing this limit should still be persisted in the database. - def patch_safe_limit_bytes - patch_hard_limit_bytes / 10 + def patch_safe_limit_bytes(limit = patch_hard_limit_bytes) + limit / 10 end # Returns the limit for a single diff file (patch). @@ -174,9 +174,13 @@ module Gitlab @line_count ||= Util.count_lines(@diff) end + def diff_bytesize + @diff_bytesize ||= @diff.bytesize + end + def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes + @too_large = diff_bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -194,7 +198,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes + @collapsed = !expanded && diff_bytesize >= self.class.patch_safe_limit_bytes end def collapse! diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index e6121d688ba..2fa88973bae 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -7,19 +7,23 @@ module Gitlab class DiffCollection include Enumerable - DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze - attr_reader :limits delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + def self.default_limits + { max_files: 100, max_lines: 5000 } + end + def self.limits(options = {}) limits = {} - limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + defaults = default_limits + limits[:max_files] = options.fetch(:max_files, defaults[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines]) limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file - limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min - limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + + limits[:safe_max_files] = [limits[:max_files], defaults[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb index 7e49d79676e..e30ec836a49 100644 --- a/lib/gitlab/git/diff_stats_collection.rb +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -22,8 +22,8 @@ module Gitlab @collection.map(&:path) end - def real_size - max_files = ::Commit.max_diff_options[:max_files] + def real_size(project: nil) + max_files = ::Commit.max_diff_options(project: project)[:max_files] if paths.size > max_files "#{max_files}+" else diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b67b3a37440..d8c992155cb 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -504,7 +504,7 @@ module Gitlab changes_size = 0 changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) # rubocop: disable CodeReuse/ActiveRecord + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) check_size_against_limit(changes_size) end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index ae83e45f2b3..b4ccca9df07 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -60,13 +60,17 @@ module Gitlab def check_valid_actor! # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 - if actor && !actor.is_a?(User) && !actor.instance_of?(Key) + if actor && !allowed_actor? raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] end super end + def allowed_actor? + actor.is_a?(User) || actor.instance_of?(Key) + end + def project_snippet? snippet.is_a?(ProjectSnippet) end @@ -138,3 +142,5 @@ module Gitlab end end end + +Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 66517ecd743..d2ddc608d3a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -45,10 +45,11 @@ module Gitlab # made globally available to the frontend push_frontend_feature_flag(:snippets_vue, default_enabled: true) push_frontend_feature_flag(:monaco_blobs, default_enabled: true) - push_frontend_feature_flag(:monaco_ci, default_enabled: false) + push_frontend_feature_flag(:monaco_ci, default_enabled: true) push_frontend_feature_flag(:snippets_edit_vue, default_enabled: true) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) + push_frontend_feature_flag(:usage_data_api, default_enabled: false) # Startup CSS feature is a special one as it can be enabled by means of cookies and params gon.push({ features: { 'startupCss' => use_startup_css? } }, true) diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb new file mode 100644 index 00000000000..a96e4c4b976 --- /dev/null +++ b/lib/gitlab/graphql/global_id_compatibility.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module GlobalIDCompatibility + # TODO: remove this module once the compatibility layer is no longer needed. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883 + def coerce_global_id_arguments!(args) + global_id_arguments = self.class.arguments.values.select do |arg| + arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType + end + + global_id_arguments.each do |arg| + k = arg.keyword + args[k] &&= arg.type.coerce_isolated_input(args[k]) + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 7be6810f7ba..0b5bde8d8d9 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -12,13 +12,19 @@ module Gitlab end method_name = kwargs.delete(:method) || name.to_s.sub(/_html$/, '') - kwargs[:resolve] = Gitlab::Graphql::MarkdownField::Resolver.new(method_name.to_sym).proc + resolver_method = "#{name}_resolver".to_sym + kwargs[:resolver_method] = resolver_method kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 field name, GraphQL::STRING_TYPE, **kwargs + + define_method resolver_method do + # We need to `dup` the context so the MarkdownHelper doesn't modify it + ::MarkupHelper.markdown_field(object, method_name.to_sym, context.to_h.dup) + end end end end diff --git a/lib/gitlab/graphql/markdown_field/resolver.rb b/lib/gitlab/graphql/markdown_field/resolver.rb deleted file mode 100644 index 11a01b95ad1..00000000000 --- a/lib/gitlab/graphql/markdown_field/resolver.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module MarkdownField - class Resolver - attr_reader :method_name - - def initialize(method_name) - @method_name = method_name - end - - def proc - -> (object, _args, ctx) do - # We need to `dup` the context so the MarkdownHelper doesn't modify it - ::MarkupHelper.markdown_field(object, method_name, ctx.to_h.dup) - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index f54695ddb9a..577f59911f5 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -94,6 +94,8 @@ module Gitlab [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] elsif ordering_by_similarity?(order_value) ['similarity', order_value.direction, order_value.expr] + elsif ordering_by_case?(order_value) + [order_value.expr.case.name.to_s, order_value.direction, order_value.expr] else [order_value.expr.name, order_value.direction, nil] end @@ -106,7 +108,12 @@ module Gitlab # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore def ordering_by_similarity?(order_value) - order_value.to_sql.match?(/SIMILARITY\(.+\*/) + Gitlab::Database::SimilarityScore.order_by_similarity?(order_value) + end + + # determine if ordering using CASE + def ordering_by_case?(order_value) + order_value.expr.is_a?(Arel::Nodes::Case) end end end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 6b6bb72eb31..1e568e9dcbc 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -6,6 +6,8 @@ module Gitlab class LoggerAnalyzer COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } + FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } + ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze def analyze?(query) Feature.enabled?(:graphql_logging, default_enabled: true) @@ -29,12 +31,13 @@ module Gitlab def final_value(memo) return if memo.nil? - analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER] - complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers) + complexity, depth, field_usages = GraphQL::Analysis.analyze_query(memo[:query], ALL_ANALYZERS) memo[:depth] = depth memo[:complexity] = complexity memo[:duration_s] = duration(memo[:time_started]).round(1) + memo[:used_fields] = field_usages.first + memo[:used_deprecated_fields] = field_usages.second GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 0cc3de297ba..751dedf4323 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -38,3 +38,5 @@ module Gitlab end end end + +Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults') diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb index cdc6d2a7519..f0c6fdab600 100644 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -22,7 +22,7 @@ module Gitlab def check return unless http_servers - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05b69362976..00c6f570f4f 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -53,6 +53,7 @@ module Gitlab @importable = importable @imported_object_retries = 0 @relation_hash[importable_column_name] = @importable.id + @original_user = {} # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' @@ -112,6 +113,7 @@ module Gitlab def update_user_references self.class::USER_REFERENCES.each do |reference| if @relation_hash[reference] + @original_user[reference] = @relation_hash[reference] @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end @@ -243,28 +245,20 @@ module Gitlab # will be used. Otherwise, a note stating the original author name # is left. def set_note_author - old_author_id = @relation_hash['author_id'] + old_author_id = @original_user['author_id'] author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) + unless @members_mapper.include?(old_author_id) + @relation_hash['note'] = "%{note}\n\n %{missing_author_note}" % { + note: @relation_hash['note'].presence || '*Blank note*', + missing_author_note: missing_author_note(@relation_hash['updated_at'], author['name']) + } + end end def missing_author_note(updated_at, author_name) timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? + "*By #{author_name} on #{timestamp} (imported from GitLab)*" end def existing_object? diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 515fd98630c..4964b8b16f4 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -21,10 +21,10 @@ module Gitlab save_lfs_object(lfs_object) end - append_lfs_json_for_batch(batch) if write_lfs_json_enabled? + append_lfs_json_for_batch(batch) end - write_lfs_json if write_lfs_json_enabled? + write_lfs_json true rescue => e @@ -35,10 +35,6 @@ module Gitlab private - def write_lfs_json_enabled? - ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) - end - def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 31d1f7b48bd..6b37683ea68 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -34,8 +34,8 @@ module Gitlab @user.id end - def include?(old_author_id) - map.has_key?(old_author_id) && map[old_author_id] != default_user_id + def include?(old_user_id) + map.has_key?(old_user_id) end private @@ -63,6 +63,8 @@ module Gitlab end def add_team_member(member, existing_user = nil) + return true if existing_user && @importable.members.exists?(user_id: existing_user.id) + member['user'] = existing_user member_hash = member_hash(member) diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 4a85a313fd7..d1ac6a55fb7 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -37,7 +37,7 @@ module Gitlab %i[get_request_count query_time read_bytes write_bytes].each do |method| define_method method do - STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord + STORAGES.sum(&method) end end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e7a8cc6305a..2cede524cac 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -23,7 +23,15 @@ module Gitlab TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total def self.notify(key, jid) - Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + Gitlab::Redis::SharedState.with do |redis| + # Use a Redis MULTI transaction to ensure we always set an expiry + redis.multi do |multi| + multi.lpush(key, jid) + # This TTL needs to be long enough to allow whichever Sidekiq job calls + # JobWaiter#wait to reach BLPOP. + multi.expire(key, 6.hours.to_i) + end + end end def self.key?(key) @@ -52,10 +60,6 @@ module Gitlab increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| - # Fallback key expiry: allow a long grace period to reduce the chance of - # a job pushing to an expired key and recreating it - redis.expire(key, [timeout * 2, 10.minutes.to_i].max) - while jobs_remaining > 0 # Redis will not take fractional seconds. Prefer waiting too long over # not waiting long enough @@ -75,9 +79,6 @@ module Gitlab @finished << jid @jobs_remaining -= 1 end - - # All jobs have finished, so expire the key immediately - redis.expire(key, 0) if jobs_remaining == 0 end finished diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index e4d600694c2..95217f86d01 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -6,6 +6,14 @@ module Gitlab # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md class Client + GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs+json' + GIT_LFS_USER_AGENT = "GitLab #{Gitlab::VERSION} LFS client" + DEFAULT_HEADERS = { + 'Accept' => GIT_LFS_CONTENT_TYPE, + 'Content-Type' => GIT_LFS_CONTENT_TYPE, + 'User-Agent' => GIT_LFS_USER_AGENT + }.freeze + attr_reader :base_url def initialize(base_url, credentials:) @@ -13,19 +21,19 @@ module Gitlab @credentials = credentials end - def batch(operation, objects) + def batch!(operation, objects) body = { operation: operation, transfers: ['basic'], # We don't know `ref`, so can't send it - objects: objects.map { |object| { oid: object.oid, size: object.size } } + objects: objects.as_json(only: [:oid, :size]) } rsp = Gitlab::HTTP.post( batch_url, basic_auth: basic_auth, body: body.to_json, - headers: { 'Content-Type' => 'application/vnd.git-lfs+json' } + headers: build_request_headers ) raise BatchSubmitError unless rsp.success? @@ -40,14 +48,15 @@ module Gitlab body end - def upload(object, upload_action, authenticated:) + def upload!(object, upload_action, authenticated:) file = object.file.open params = { body_stream: file, headers: { 'Content-Length' => object.size.to_s, - 'Content-Type' => 'application/octet-stream' + 'Content-Type' => 'application/octet-stream', + 'User-Agent' => GIT_LFS_USER_AGENT }.merge(upload_action['header'] || {}) } @@ -60,8 +69,25 @@ module Gitlab file&.close end + def verify!(object, verify_action, authenticated:) + params = { + body: object.to_json(only: [:oid, :size]), + headers: build_request_headers(verify_action['header']) + } + + params[:basic_auth] = basic_auth unless authenticated + + rsp = Gitlab::HTTP.post(verify_action['href'], params) + + raise ObjectVerifyError unless rsp.success? + end + private + def build_request_headers(extra_headers = nil) + DEFAULT_HEADERS.merge(extra_headers || {}) + end + attr_reader :credentials def batch_url @@ -96,6 +122,12 @@ module Gitlab "Failed to upload object" end end + + class ObjectVerifyError < StandardError + def message + "Failed to verify object" + end + end end end end diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb new file mode 100644 index 00000000000..80dff075391 --- /dev/null +++ b/lib/gitlab/manifest_import/metadata.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module ManifestImport + class Metadata + EXPIRY_TIME = 1.week + + attr_reader :user, :fallback + + def initialize(user, fallback: {}) + @user = user + @fallback = fallback + end + + def save(repositories, group_id) + Gitlab::Redis::SharedState.with do |redis| + redis.multi do + redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + end + end + end + + def repositories + redis_get('repositories').then do |repositories| + next unless repositories + + Gitlab::Json.parse(repositories).map(&:symbolize_keys) + end || fallback[:manifest_import_repositories] + end + + def group_id + redis_get('group_id')&.to_i || fallback[:manifest_import_group_id] + end + + private + + def key_for(field) + "manifest_import:metadata:user:#{user.id}:#{field}" + end + + def redis_get(field) + Gitlab::Redis::SharedState.with do |redis| + redis.get(key_for(field)) + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index d1490d5d9b6..8a176be30a2 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -13,11 +13,12 @@ module Gitlab @dashboard_hash = dashboard_hash @project = project @dashboard_path = dashboard_path + @affected_environment_ids = [] end def execute import - rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError false end @@ -32,28 +33,51 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics + update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord def create_or_update_metrics # TODO: use upsert and worker for callbacks? + + affected_metric_ids = [] prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + + affected_metric_ids << prometheus_metric.id end + + @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord def delete_stale_metrics - identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } stale_metrics = PrometheusMetric.for_project(project) .for_dashboard_path(dashboard_path) .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers) + .not_identifier(identifiers_from_yml) + + return unless stale_metrics.exists? + + delete_stale_alerts(stale_metrics) + stale_metrics.each_batch { |batch| batch.delete_all } + end + + def delete_stale_alerts(stale_metrics) + stale_alerts = find_alerts(stale_metrics) + + affected_environment_ids = stale_alerts.get_environment_id + return unless affected_environment_ids.present? - # TODO: use destroy_all and worker for callbacks? - stale_metrics.each(&:destroy) + @affected_environment_ids += affected_environment_ids + stale_alerts.each_batch { |batch| batch.delete_all } + end + + def find_alerts(metrics) + Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute end def prometheus_metrics_attributes @@ -65,6 +89,19 @@ module Gitlab ).execute end end + + def update_prometheus_environments + affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) + + return unless affected_environments.exists? + + affected_environments.each do |affected_environment| + ::Clusters::Applications::ScheduleUpdateService.new( + affected_environment.cluster_prometheus_adapter, + project + ).execute + end + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb new file mode 100644 index 00000000000..5ed4466f440 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + # Acts on metrics which have been ingested from source controlled dashboards + class CustomDashboardMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, includes + # the record's id in the dashboard config. + def transform! + database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute + + for_metrics do |metric| + metric_record = database_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb index 9e2bb0d1a70..ad9d78133af 100644 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -46,7 +46,7 @@ module Gitlab links&.each do |link| next unless link.is_a? Hash - Gitlab::UrlBlocker.validate!(link[:url], blocker_args) + Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) rescue Gitlab::UrlBlocker::BlockedUrlError link[:url] = '' end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb index 4d94ab098ae..bc85dc4e131 100644 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -4,10 +4,10 @@ module Gitlab module Metrics module Dashboard module Transformers - TransformerError = Class.new(StandardError) - module Errors - class MissingAttribute < TransformerError + BaseError = Class.new(StandardError) + + class MissingAttribute < BaseError def initialize(attribute_name) super("Missing attribute: '#{attribute_name}'") end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 15db3999fa4..f8ba254c2a7 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -4,15 +4,13 @@ module Gitlab module Metrics class RequestsRackMiddleware HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503), - "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503), + "delete" => %w(200 202 204 303 400 401 403 404 500 503), + "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), + "head" => %w(200 204 301 302 303 401 403 404 410 500), "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 422 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503), - "propfind" => %w(404), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500), - "report" => %w(404) + "patch" => %w(200 202 204 400 403 404 409 416 500), + "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), + "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) }.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze @@ -48,10 +46,12 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase + method = 'INVALID' unless HTTP_METHODS.key?(method) started = Time.now.to_f + health_endpoint = health_endpoint?(env['PATH_INFO']) begin - if health_endpoint?(env['PATH_INFO']) + if health_endpoint RequestsRackMiddleware.http_health_requests_total.increment(method: method) else RequestsRackMiddleware.http_request_total.increment(method: method) @@ -60,7 +60,10 @@ module Gitlab status, headers, body = @app.call(env) elapsed = Time.now.to_f - started - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + + unless health_endpoint + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + end [status, headers, body] rescue diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 8c4d150adad..d7935d65e12 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -54,7 +54,7 @@ module Gitlab end def unicorn_workers_count - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 8e6ac7610f2..e7e18b3bb82 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -137,6 +137,7 @@ module Gitlab # TODO this class is meant to replace Handler when the feature flag # upload_middleware_jwt_params_handler is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps class HandlerForJWTParams < Handler def with_open_files @rewritten_fields.keys.each do |field| diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 8796dd4d7ec..46c74b8fe3c 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -10,9 +10,9 @@ module Gitlab @request_context = request_context end - def paginate(relation) + def paginate(relation, exclude_total_headers: false) paginate_with_limit_optimization(add_default_order(relation)).tap do |data| - add_pagination_headers(data) + add_pagination_headers(data, exclude_total_headers) end end @@ -27,7 +27,7 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT @@ -47,14 +47,14 @@ module Gitlab relation end - def add_pagination_headers(paginated_data) + def add_pagination_headers(paginated_data, exclude_total_headers) header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) - return if data_without_counts?(paginated_data) + return if exclude_total_headers || data_without_counts?(paginated_data) header 'X-Total', paginated_data.total_count.to_s header 'X-Total-Pages', total_pages(paginated_data).to_s diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index e6e599e079d..fa3af269bbf 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -52,6 +52,7 @@ module Gitlab ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'), + ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 98db8ff761e..c8c949a9363 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -121,6 +121,20 @@ module Gitlab result[:message] end end + + desc _('Approve a merge request') + explanation _('Approve the current merge request.') + types MergeRequest + condition do + quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) + end + command :approve do + success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target) + + next unless success + + @execution_message[:approve] = _('Approved the current merge request.') + end end def merge_orchestration_service diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index ff5754675e2..496018c88cb 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -7,11 +7,11 @@ module Gitlab KeyFormatError = Class.new(StandardError) def self.count(params) - self.new.count(params) + self.new.count(**params) end def self.add(params) - self.new.add(params) + self.new.add(**params) end def count(keys:) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8e23ac6aca5..6511b84e947 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,6 +61,39 @@ module Gitlab )\z}xi.freeze end + def debian_package_name_regex + # See official parser + # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122 + # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze + # But we prefer a more strict version from Lintian + # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 + @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze + end + + def debian_version_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 + @debian_version_regex ||= %r{ + \A(?: + (?:([0-9]{1,9}):)? (?# epoch) + ([0-9][0-9a-z\.+~-]*) (?# version) + (?:(-[0-0a-z\.+~]+))? (?# revision) + )\z}xi.freeze + end + + def debian_architecture_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 + # But we limit to lower case + @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze + end + + def debian_distribution_regex + @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze + end + + def debian_component_regex + @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -80,6 +113,11 @@ module Gitlab @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end + def prefixed_semver_regex + # identical to semver_regex, except starting with 'v' + @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + def go_package_regex # A Go package name looks like a URL but is not; it: # - Must not have a scheme, such as http:// or https:// @@ -103,6 +141,14 @@ module Gitlab def generic_package_version_regex /\A\d+\.\d+\.\d+\z/ end + + def generic_package_name_regex + maven_file_name_regex + end + + def generic_package_file_name_regex + generic_package_name_regex + end end extend self @@ -211,8 +257,27 @@ module Gitlab "Must start with a letter, and cannot end with '-'" end + # The section start, e.g. section_start:12345678:NAME + def logs_section_prefix_regex + /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)/ + end + + # The optional section options, e.g. [collapsed=true] + def logs_section_options_regex + /(\[(?:\w+=\w+)(?:, ?(?:\w+=\w+))*\])?/ + end + + # The region end, always: \r\e\[0K + def logs_section_suffix_regex + /\r\033\[0K/ + end + def build_trace_section_regex - @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze + @build_trace_section_regexp ||= %r{ + #{logs_section_prefix_regex} + #{logs_section_options_regex} + #{logs_section_suffix_regex} + }x.freeze end def markdown_code_or_html_blocks diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index cd03a347355..8f5495ece5e 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -131,12 +131,12 @@ module Gitlab def shift_left move_sequence_before(true) - object.reset + object.reset_relative_position end def shift_right move_sequence_after(true) - object.reset + object.reset_relative_position end def create_space_left diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 06d8dca2f70..1e78464ddf8 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :filters + attr_reader :current_user, :query, :sort, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,31 +19,23 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @sort = sort @filters = filters end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil) should_preload = preload_method.present? - collection = case scope - when 'projects' - projects - when 'issues' - issues - when 'merge_requests' - merge_requests - when 'milestones' - milestones - when 'users' - users - else - should_preload = false - Kaminari.paginate_array([]) - end + collection = collection_for(scope) + + if collection.nil? + should_preload = false + collection = Kaminari.paginate_array([]) + end collection = collection.public_send(preload_method) if should_preload # rubocop:disable GitlabSecurity/PublicSend collection = collection.page(page).per(per_page) @@ -118,6 +110,34 @@ module Gitlab private + def collection_for(scope) + case scope + when 'projects' + projects + when 'issues' + issues + when 'merge_requests' + merge_requests + when 'milestones' + milestones + when 'users' + users + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_sort(scope) + case sort + when 'oldest' + scope.reorder('created_at ASC') + when 'newest' + scope.reorder('created_at DESC') + else + scope + end + end + # rubocop: enable CodeReuse/ActiveRecord + def projects limit_projects.search(query) end @@ -129,7 +149,7 @@ module Gitlab issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - issues + apply_sort(issues) end # rubocop: disable CodeReuse/ActiveRecord @@ -149,7 +169,7 @@ module Gitlab merge_requests = merge_requests.in_projects(project_ids_relation) end - merge_requests + apply_sort(merge_requests) end def default_scope @@ -193,6 +213,10 @@ module Gitlab end params[:state] = filters[:state] if filters.key?(:state) + + if Feature.enabled?(:search_filter_by_confidential) && filters.key?(:confidential) && %w(yes no).include?(filters[:confidential]) + params[:confidential] = filters[:confidential] == 'yes' + end end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index e1a87a77f04..8793a672693 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -231,7 +231,7 @@ module Gitlab def rss_increase_by_jobs Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| rss_increase_by_job(job) end end diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index f647c85e1c8..315c603c1dd 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -3,11 +3,38 @@ module Gitlab module StaticSiteEditor module Config + # + # Base GitLab Static Site Editor Configuration facade + # class FileConfig - def data - { - static_site_generator: 'middleman' - } + ConfigError = Class.new(StandardError) + + def initialize(yaml) + content_hash = content_hash(yaml) + @global = Entry::Global.new(content_hash) + @global.compose! + rescue Gitlab::Config::Loader::FormatError => e + raise FileConfig::ConfigError, e.message + end + + def valid? + @global.valid? + end + + def errors + @global.errors + end + + def to_hash_with_defaults + # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') + # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. + @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h + end + + private + + def content_hash(yaml) + Gitlab::Config::Loader::Yaml.new(yaml).load! end end end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb new file mode 100644 index 00000000000..c295ccf1d11 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # This class represents a global entry - root Entry for entire + # GitLab StaticSiteEditor Configuration file. + # + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[ + image_upload_path + mounts + static_site_generator + ].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :image_upload_path, Entry::ImageUploadPath, + description: 'Configuration of the Static Site Editor image upload path.' + entry :mounts, Entry::Mounts, + description: 'Configuration of the Static Site Editor mounts.' + entry :static_site_generator, Entry::StaticSiteGenerator, + description: 'Configuration of the Static Site Editor static site generator.' + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb new file mode 100644 index 00000000000..6a2b9e10d33 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the path to which images will be uploaded + # + class ImageUploadPath < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + + def self.default + 'source/images' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb new file mode 100644 index 00000000000..b10956e17a5 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mount < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[source target].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :source, type: String, presence: true + validates :target, type: String, presence: true, allow_blank: true + end + + def self.default + # NOTE: This is the default for middleman projects. Ideally, this would be determined + # based on the defaults for whatever `static_site_generator` is configured. + { + source: 'source', + target: '' + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb new file mode 100644 index 00000000000..10bd377e419 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mounts < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + + entry :mount, Entry::Mount, description: 'Configuration of a Static Site Editor mount.' + + validations do + validates :config, type: Array, presence: true + end + + def skip_config_hash_validation? + true + end + + def self.default + [Entry::Mount.default] + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb new file mode 100644 index 00000000000..593c0951f93 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the static site generator tool/framework. + # + class StaticSiteGenerator < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String, inclusion: { in: %w[middleman], message: "should be 'middleman'" } + end + + def self.default + 'middleman' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index f3dce74a32f..ff24ec69ab0 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -4,8 +4,6 @@ module Gitlab module StaticSiteEditor module Config class GeneratedConfig - SUPPORTED_EXTENSIONS = %w[.md].freeze - def initialize(repository, ref, path, return_url) @repository = repository @ref = ref @@ -23,7 +21,7 @@ module Gitlab project: project.path, namespace: project.namespace.full_path, return_url: sanitize_url(return_url), - is_supported_content: supported_content?.to_s, + is_supported_content: supported_content?, base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), merge_requests_illustration_path: merge_requests_illustration_path } @@ -35,8 +33,12 @@ module Gitlab delegate :project, to: :repository + def supported_extensions + %w[.md].freeze + end + def commit_id - repository.commit(ref)&.id if ref + repository.commit(ref)&.id end def supported_content? @@ -50,7 +52,7 @@ module Gitlab def extension_supported? return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - SUPPORTED_EXTENSIONS.any? { |ext| path.end_with?(ext) } + supported_extensions.any? { |ext| path.end_with?(ext) } end def file_exists? diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 72783a2d682..16e7b8a7eca 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,21 +10,21 @@ module Gitlab APPLICATION_DEFAULT = 1 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class) + Theme = Struct.new(:id, :name, :css_class, :css_filename) # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo'), - Theme.new(4, 'Blue', 'ui-blue'), - Theme.new(7, 'Light Blue', 'ui-light-blue'), - Theme.new(5, 'Green', 'ui-green'), - Theme.new(8, 'Light Green', 'ui-light-green'), - Theme.new(9, 'Red', 'ui-red'), - Theme.new(10, 'Light Red', 'ui-light-red'), - Theme.new(2, 'Dark', 'ui-dark'), - Theme.new(3, 'Light', 'ui-light'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark') + Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'), + Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'), + Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'), + Theme.new(5, 'Green', 'ui-green', 'theme_green'), + Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'), + Theme.new(9, 'Red', 'ui-red', 'theme_red'), + Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'), + Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'), + Theme.new(3, 'Light', 'ui-light', 'theme_light'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil) ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 89605ce5d07..fbfc7beed9a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -138,8 +138,10 @@ module Gitlab pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), + projects_creating_incidents: distinct_count(Issue.incident, :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), + projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -166,8 +168,7 @@ module Gitlab user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts, - snowplow_event_counts + service_desk_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -175,7 +176,7 @@ module Gitlab end # rubocop: enable Metrics/AbcSize - def snowplow_event_counts(time_period: {}) + def snowplow_event_counts(time_period) return {} unless report_snowplow_events? { @@ -242,7 +243,8 @@ module Gitlab signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), - grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? } + grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, + gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } end @@ -444,8 +446,11 @@ module Gitlab # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord + # augmented in EE def user_preferences_usage - {} # augmented in EE + { + user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active)) + } end def merge_requests_users(time_period) @@ -469,7 +474,7 @@ module Gitlab end def last_28_days_time_period(column: :created_at) - { column => 28.days.ago..Time.current } + { column => 30.days.ago..2.days.ago } end # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv @@ -541,6 +546,7 @@ module Gitlab groups: distinct_count(::GroupMember.where(time_period), :user_id), users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, + user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), projects_imported: { gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), @@ -555,7 +561,8 @@ module Gitlab jira: distinct_count(::JiraImportState.where(time_period), :user_id), fogbugz: projects_imported_count('fogbugz', time_period), phabricator: projects_imported_count('phabricator', time_period) - } + }, + groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -567,7 +574,8 @@ module Gitlab clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: user_minimum_id, - finish: user_maximum_id) + finish: user_maximum_id), + projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -696,10 +704,10 @@ module Gitlab counter = Gitlab::UsageDataCounters::EditorUniqueCounter { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) } + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } } end @@ -812,6 +820,7 @@ module Gitlab clear_memoization(:approval_merge_request_rule_maximum_id) clear_memoization(:project_minimum_id) clear_memoization(:project_maximum_id) + clear_memoization(:auth_providers) end # rubocop: disable CodeReuse/ActiveRecord @@ -843,6 +852,39 @@ module Gitlab def projects_imported_count(from, time_period) distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord end + + # rubocop:disable CodeReuse/ActiveRecord + def distinct_count_user_auth_by_provider(time_period) + counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| + hash[provider] = distinct_count( + ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id) + end + + if any_ldap_auth_providers? + counts['ldap'] = distinct_count( + ::AuthenticationEvent.success.ldap.where(time_period), :user_id + ) + end + + counts + end + # rubocop:enable CodeReuse/ActiveRecord + + # rubocop:disable UsageData/LargeTable + def auth_providers + strong_memoize(:auth_providers) do + ::AuthenticationEvent.providers + end + end + # rubocop:enable UsageData/LargeTable + + def auth_providers_except_ldap + auth_providers.reject { |provider| provider.starts_with?('ldap') } + end + + def any_ldap_auth_providers? + auth_providers.any? { |provider| provider.starts_with?('ldap') } + end end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 53bf6daea4c..eb132ef0967 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -72,7 +72,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index fc1b5a59487..b1ce46a1ff5 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -3,14 +3,26 @@ module Gitlab module UsageDataCounters module IssueActivityUniqueCounter - ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_CATEGORY = 'issues_edit' + ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed' + ISSUE_CREATED = 'g_project_management_issue_created' + ISSUE_CLOSED = 'g_project_management_issue_closed' + ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' + ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' - ISSUE_CATEGORY = 'issues_edit' + ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' + ISSUE_REOPENED = 'g_project_management_issue_reopened' + ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' + ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' class << self + def track_issue_created_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CREATED, author, time) + end + def track_issue_title_changed_action(author:, time: Time.zone.now) track_unique_action(ISSUE_TITLE_CHANGED, author, time) end @@ -31,6 +43,30 @@ module Gitlab track_unique_action(ISSUE_MADE_VISIBLE, author, time) end + def track_issue_closed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CLOSED, author, time) + end + + def track_issue_reopened_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_REOPENED, author, time) + end + + def track_issue_label_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_LABEL_CHANGED, author, time) + end + + def track_issue_milestone_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) + end + + def track_issue_iteration_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ITERATION_CHANGED, author, time) + end + + def track_issue_weight_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) + end + private def track_unique_action(action, author, time) diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index 25e7f858bb1..9bbf804d87c 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -206,3 +206,31 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_created + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_closed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_reopened + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_label_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_milestone_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_iteration_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_weight_changed + category: issues_edit + redis_slot: project_management + aggregation: daily diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index bacd63ab282..c54e766230e 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -22,7 +22,7 @@ module Gitlab end def sum(relation, column, *rest) - relation.select(relation.all.table[column].sum).to_sql # rubocop:disable CodeReuse/ActiveRecord + relation.select(relation.all.table[column].sum).to_sql end private diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index ca6a36c9cea..5267733d220 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,9 +39,9 @@ module Gitlab FALLBACK = -1 - def count(relation, column = nil, batch: true, start: nil, finish: nil) + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch - Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) + Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) else relation.count end @@ -106,7 +106,6 @@ module Gitlab # @param values [Array|String] the values counted def track_usage_event(event_name, values) return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - return unless Gitlab::CurrentSettings.usage_ping_enabled? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) end diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index d2c01bbd55e..6a8a7e24ebd 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -1,16 +1,33 @@ # frozen_string_literal: true -require 'webpack/rails/manifest' +require 'net/http' +require 'uri' module Gitlab module Webpack - class Manifest < ::Webpack::Rails::Manifest - # Raised if a supplied asset does not exist in the webpack manifest + class Manifest + # Raised if we can't read our webpack manifest for whatever reason + class ManifestLoadError < StandardError + def initialize(message, orig) + super "#{message} (original error #{orig})" + end + end + + # Raised if webpack couldn't build one of your entry points + class WebpackError < StandardError + def initialize(errors) + super "Error in webpack compile, details follow below:\n#{errors.join("\n\n")}" + end + end + + # Raised if a supplied entry point does not exist in the webpack manifest AssetMissingError = Class.new(StandardError) class << self + include Gitlab::Utils::StrongMemoize + def entrypoint_paths(source) - raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled? + raise WebpackError, manifest["errors"] unless manifest_bundled? dll_assets = manifest.fetch("dllAssets", []) entrypoint = manifest["entrypoints"][source] @@ -21,9 +38,86 @@ module Gitlab "/#{::Rails.configuration.webpack.public_path}/#{p}" end else + raise AssetMissingError, "Can't find asset '#{source}' in webpack manifest" + end + end + + def asset_paths(source) + raise WebpackError, manifest["errors"] unless manifest_bundled? + + paths = manifest["assetsByChunkName"][source] + if paths + # Can be either a string or an array of strings. + # Do not include source maps as they are not javascript + [paths].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| + "/#{::Rails.configuration.webpack.public_path}/#{p}" + end + else raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest" end end + + def clear_manifest! + clear_memoization(:manifest) + end + + private + + def manifest_bundled? + !manifest["errors"].any? { |error| error.include? "Module build failed" } + end + + def manifest + if ::Rails.configuration.webpack.dev_server.enabled + # Don't cache if we're in dev server mode, manifest may change ... + load_manifest + else + # ... otherwise cache at class level, as JSON loading/parsing can be expensive + strong_memoize(:manifest) { load_manifest } + end + end + + def load_manifest + data = if ::Rails.configuration.webpack.dev_server.enabled + load_dev_server_manifest + else + load_static_manifest + end + + Gitlab::Json.parse(data) + end + + def load_dev_server_manifest + host = ::Rails.configuration.webpack.dev_server.manifest_host + port = ::Rails.configuration.webpack.dev_server.manifest_port + uri = Addressable::URI.new(scheme: 'http', host: host, port: port, path: dev_server_path) + + # localhost could be blocked via Gitlab::HTTP + response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty + + return response.body if response.code == 200 + + raise "HTTP error #{response.code}" + rescue => e + raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri} - is it running, and is stats-webpack-plugin loaded?", e) + end + + def load_static_manifest + File.read(static_manifest_path) + rescue => e + raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path} - have you run `rake webpack:compile`?", e) + end + + def static_manifest_path + ::Rails.root.join( + ::Rails.configuration.webpack.output_dir, + ::Rails.configuration.webpack.manifest_filename + ) + end + + def dev_server_path + "/#{::Rails.configuration.webpack.public_path}/#{::Rails.configuration.webpack.manifest_filename}" + end end end end diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index a906beda80e..acce9a455bd 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -11,7 +11,7 @@ class GitlabDanger karma database commit_messages - telemetry + product_analytics utility_css pajamas ].freeze diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb index b419f79bace..7c0e56b61c8 100644 --- a/lib/grafana/client.rb +++ b/lib/grafana/client.rb @@ -19,8 +19,8 @@ module Grafana # @param name [String] Unique identifier for a Grafana datasource def get_datasource(name:) # CGI#escape formats strings such that the Grafana endpoint - # will not recognize the dashboard name. Preferring URI#escape. - http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape + # will not recognize the dashboard name. Prefer Addressable::URI#encode_component. + http_get("#{@api_url}/api/datasources/name/#{Addressable::URI.encode_component(name)}") end # @param datasource_id [String] Grafana ID for the datasource diff --git a/lib/pager_duty/validator/schemas/message.json b/lib/pager_duty/validator/schemas/message.json new file mode 100644 index 00000000000..b1a3185cd1a --- /dev/null +++ b/lib/pager_duty/validator/schemas/message.json @@ -0,0 +1,47 @@ +{ + "type": "object", + "required": ["event", "incident"], + "properties": { + "event": { "type": "string" }, + "incident": { + "type": "object", + "required": [ + "html_url", + "incident_number", + "title", + "status", + "created_at", + "urgency", + "incident_key" + ], + "properties": { + "html_url": { "type": "string" }, + "incindent_number": { "type": "integer" }, + "title": { "type": "string" }, + "status": { "type": "string" }, + "created_at": { "type": "string" }, + "urgency": { "type": "string", "enum": ["high", "low"] }, + "incident_key": { "type": ["string", "null"] }, + "assignments": { + "type": "array", + "items": { + "assignee": { + "type": "array", + "items": { + "summary": { "type": "string" }, + "html_url": { "type": "string" } + } + } + } + }, + "impacted_services": { + "type": "array", + "items": { + "summary": { "type": "string" }, + "html_url": { "type": "string" } + } + } + } + } + } +} diff --git a/lib/pager_duty/webhook_payload_parser.rb b/lib/pager_duty/webhook_payload_parser.rb index 573fb36f0ca..11071926cf2 100644 --- a/lib/pager_duty/webhook_payload_parser.rb +++ b/lib/pager_duty/webhook_payload_parser.rb @@ -2,6 +2,8 @@ module PagerDuty class WebhookPayloadParser + SCHEMA_PATH = File.join('lib', 'pager_duty', 'validator', 'schemas', 'message.json') + def initialize(payload) @payload = payload end @@ -11,7 +13,7 @@ module PagerDuty end def call - Array(payload['messages']).map { |msg| parse_message(msg) } + Array(payload['messages']).map { |msg| parse_message(msg) }.reject(&:empty?) end private @@ -19,6 +21,8 @@ module PagerDuty attr_reader :payload def parse_message(message) + return {} unless valid_message?(message) + { 'event' => message['event'], 'incident' => parse_incident(message['incident']) @@ -26,8 +30,6 @@ module PagerDuty end def parse_incident(incident) - return {} if incident.blank? - { 'url' => incident['html_url'], 'incident_number' => incident['incident_number'], @@ -62,5 +64,9 @@ module PagerDuty def reject_empty(entities) Array(entities).reject { |e| e['summary'].blank? && e['url'].blank? } end + + def valid_message?(message) + ::JSONSchemer.schema(Pathname.new(SCHEMA_PATH)).valid?(message) + end end end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb index 389f5301079..4cc2e85c7bb 100644 --- a/lib/peek/views/detailed_view.rb +++ b/lib/peek/views/detailed_view.rb @@ -23,7 +23,7 @@ module Peek private def duration - detail_store.map { |entry| entry[:duration] }.sum * 1000 # rubocop:disable CodeReuse/ActiveRecord + detail_store.map { |entry| entry[:duration] }.sum * 1000 end def calls diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb index 679c021c730..ac33f0b3c2c 100644 --- a/lib/safe_zip/extract.rb +++ b/lib/safe_zip/extract.rb @@ -19,11 +19,7 @@ module SafeZip def extract(opts = {}) params = SafeZip::ExtractParams.new(**opts) - if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true) - extract_with_ruby_zip(params) - else - legacy_unsafe_extract_with_system_zip(params) - end + extract_with_ruby_zip(params) end private @@ -53,21 +49,5 @@ module SafeZip .extract end end - - def legacy_unsafe_extract_with_system_zip(params) - # Requires UnZip at least 6.00 Info-ZIP. - # -n never overwrite existing files - args = %W(unzip -n -qq #{archive_path}) - - # We add * to end of directory, because we want to extract directory and all subdirectories - args += params.directories_wildcard - - # Target directory where we extract - args += %W(-d #{params.extract_path}) - - unless system(*args) - raise Error, 'archive failed to extract' - end - end end end diff --git a/lib/sentry/client/issue.rb b/lib/sentry/client/issue.rb index c5e9df9cd21..f714bda49fd 100644 --- a/lib/sentry/client/issue.rb +++ b/lib/sentry/client/issue.rb @@ -14,7 +14,7 @@ module Sentry }.freeze def list_issues(**keyword_args) - response = get_issues(keyword_args) + response = get_issues(**keyword_args) issues = response[:issues] pagination = response[:pagination] @@ -44,7 +44,7 @@ module Sentry def get_issues(**keyword_args) response = http_get( api_urls.issues_url, - query: list_issue_sentry_query(keyword_args) + query: list_issue_sentry_query(**keyword_args) ) { diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index caa583fb3a9..ab2d77eeaf0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -81,7 +81,7 @@ namespace :gitlab do if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists - Rake::Task['webpack:compile'].invoke + system('yarn webpack') end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2a3713ed85c..de2dfca8c1b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -107,7 +107,7 @@ namespace :gitlab do puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) exit 1 else - Backup::Repository.new(progress).dump( + Backup::Repositories.new(progress).dump( max_concurrency: max_concurrency, max_storage_concurrency: max_storage_concurrency ) @@ -117,7 +117,7 @@ namespace :gitlab do task restore: :gitlab_environment do puts_time "Restoring repositories ...".color(:blue) - Backup::Repository.new(progress).restore + Backup::Repositories.new(progress).restore puts_time "done".color(:green) end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 425f66918b0..8a1809f9dfc 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -35,6 +35,11 @@ namespace :gitlab do # Truncate schema_migrations to ensure migrations re-run connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' + # Drop any views + connection.views.each do |view| + connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") + end + # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html # Add `IF EXISTS` because cascade could have already deleted a table. @@ -169,9 +174,21 @@ namespace :gitlab do desc 'reindex a regular (non-unique) index without downtime to eliminate bloat' task :reindex, [:index_name] => :environment do |_, args| - raise ArgumentError, 'must give the index name to reindex' unless args[:index_name] + unless Feature.enabled?(:database_reindexing, type: :ops) + puts "This feature (database_reindexing) is currently disabled.".color(:yellow) + exit + end + + indexes = if args[:index_name] + Gitlab::Database::PostgresIndex.by_identifier(args[:index_name]) + else + Gitlab::Database::Reindexing.candidate_indexes.random_few(2) + end - Gitlab::Database::ConcurrentReindex.new(args[:index_name], logger: Logger.new(STDOUT)).execute + Gitlab::Database::Reindexing.perform(indexes) + rescue => e + Gitlab::AppLogger.error(e) + raise end end end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index cd5943b552e..9b034d1c6c2 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -42,6 +42,9 @@ class UploadedFile @remote_id = remote_id end + # TODO this function is meant to replace .from_params when the feature flag + # upload_middleware_jwt_params_handler is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps def self.from_params_without_field(params, upload_paths) path = params['path'] remote_id = params['remote_id'] @@ -68,6 +71,10 @@ class UploadedFile ) end + # Deprecated. Don't use it. + # .from_params_without_field will replace this one + # See .from_params_without_field and + # https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps def self.from_params(params, field, upload_paths, path_override = nil) path = path_override || params["#{field}.path"] remote_id = params["#{field}.remote_id"] |