diff options
author | Rémy Coutable <remy@rymai.me> | 2019-02-04 17:11:04 +0300 |
---|---|---|
committer | Rémy Coutable <remy@rymai.me> | 2019-02-04 17:11:04 +0300 |
commit | 5c583c8e87f6b299b3a7ea07dded0972740092e9 (patch) | |
tree | 63fda670aa22f49d062e23e10dc13f9983e2d5f1 /lib | |
parent | cc41a771832a9d44c9a87e25aa784cb904d03fd5 (diff) | |
parent | 6b0b14f81d6def6d74b303cd27fef6f98aaabfd0 (diff) |
Merge branch 'master' into '46448-add-timestamps-for-each-stage-of-gitlab-rake-gitlab-backup-restore'
# Conflicts:
# spec/tasks/gitlab/backup_rake_spec.rb
Diffstat (limited to 'lib')
1400 files changed, 27187 insertions, 10579 deletions
diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb deleted file mode 100644 index 3cb1694b9f1..00000000000 --- a/lib/additional_email_headers_interceptor.rb +++ /dev/null @@ -1,6 +0,0 @@ -class AdditionalEmailHeadersInterceptor - def self.delivering_email(message) - message.header['Auto-Submitted'] ||= 'auto-generated' - message.header['X-Auto-Response-Suppress'] ||= 'All' - end -end diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index a4d8507960e..6fb7985f955 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module AfterCommitQueue extend ActiveSupport::Concern diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index ae13c248171..ee8dc822098 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AccessRequests < Grape::API include PaginationParams @@ -10,7 +12,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester @@ -18,6 +20,7 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/access_requests" do source = find_source(source_type, params[:id]) @@ -26,6 +29,7 @@ module API present access_requesters, with: Entities::AccessRequester end + # rubocop: enable CodeReuse/ActiveRecord desc "Requests access for the authenticated user to a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' @@ -50,6 +54,7 @@ module API requires :user_id, type: Integer, desc: 'The user ID of the access requester' optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/access_requests/:user_id/approve' do source = find_source(source_type, params[:id]) @@ -61,6 +66,7 @@ module API status :created present member, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Denies an access request for the given user.' do detail 'This feature was introduced in GitLab 8.11.' @@ -68,6 +74,7 @@ module API params do requires :user_id, type: Integer, desc: 'The user ID of the access requester' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/access_requests/:user_id" do source = find_source(source_type, params[:id]) member = source.requesters.find_by!(user_id: params[:user_id]) @@ -76,6 +83,7 @@ module API ::Members::DestroyService.new(current_user).execute(member) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 7ea575a9661..9cbfc0e35ff 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class API < Grape::API include APIGuard @@ -5,8 +7,9 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") NO_SLASH_URL_PART_REGEX = %r{[^/]+} - PROJECT_ENDPOINT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze - COMMIT_ENDPOINT_REQUIREMENTS = PROJECT_ENDPOINT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze + COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze + USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze insert_before Grape::Middleware::Error, GrapeLogging::Middleware::RequestLogger, @@ -15,8 +18,11 @@ module API include: [ GrapeLogging::Loggers::FilterParameters.new, GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::RouteLogger.new, Gitlab::GrapeLogging::Loggers::UserLogger.new, - Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new + Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, + Gitlab::GrapeLogging::Loggers::PerfLogger.new, + Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new ] allow_access_with_scope :api @@ -46,6 +52,10 @@ module API rack_response({ 'message' => '404 Not found' }.to_json, 404) end + rescue_from ::Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError do + rack_response({ 'message' => '409 Conflict: Resource lock' }.to_json, 409) + end + rescue_from UploadedFile::InvalidPathError do |e| rack_response({ 'message' => e.message }.to_json, 400) end @@ -76,13 +86,13 @@ module API content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers - helpers ::SentryHelper helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers # Keep in alphabetical order mount ::API::AccessRequests mount ::API::Applications + mount ::API::Avatar mount ::API::AwardEmoji mount ::API::Badges mount ::API::Boards @@ -91,6 +101,7 @@ module API mount ::API::CircuitBreakers mount ::API::Commits mount ::API::CommitStatuses + mount ::API::ContainerRegistry mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments @@ -98,12 +109,14 @@ module API mount ::API::Features mount ::API::Files mount ::API::GroupBoards - mount ::API::Groups mount ::API::GroupMilestones + mount ::API::Groups + mount ::API::GroupVariables + mount ::API::ImportGithub mount ::API::Internal mount ::API::Issues - mount ::API::Jobs mount ::API::JobArtifacts + mount ::API::Jobs mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -114,18 +127,24 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::Discussions + mount ::API::ResourceLabelEvents mount ::API::NotificationSettings mount ::API::PagesDomains mount ::API::Pipelines mount ::API::PipelineSchedules + mount ::API::ProjectClusters mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks - mount ::API::Projects mount ::API::ProjectMilestones + mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets + mount ::API::ProjectTemplates mount ::API::ProtectedBranches + mount ::API::ProtectedTags + mount ::API::Releases + mount ::API::Release::Links mount ::API::Repositories mount ::API::Runner mount ::API::Runners @@ -134,7 +153,9 @@ module API mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::Snippets + mount ::API::Submodules mount ::API::Subscriptions + mount ::API::Suggestions mount ::API::SystemHooks mount ::API::Tags mount ::API::Templates @@ -142,7 +163,6 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables - mount ::API::GroupVariables mount ::API::Version mount ::API::Wikis diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index c17089759de..af9b519ed9e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Guard API with OAuth 2.0 Access Token require 'rack/oauth2' @@ -84,7 +86,7 @@ module API end end - module ClassMethods + class_methods do private def install_error_responders(base) @@ -92,6 +94,7 @@ module API Gitlab::Auth::TokenNotFoundError, Gitlab::Auth::ExpiredError, Gitlab::Auth::RevokedError, + Gitlab::Auth::ImpersonationDisabled, Gitlab::Auth::InsufficientScopeError] base.__send__(:rescue_from, *error_classes, oauth2_bearer_token_error_handler) # rubocop:disable GitlabSecurity/PublicSend @@ -119,6 +122,11 @@ module API :invalid_token, "Token was revoked. You have to re-authorize from the user.") + when Gitlab::Auth::ImpersonationDisabled + Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new( + :invalid_token, + "Token is an impersonation token but impersonation was disabled.") + when Gitlab::Auth::InsufficientScopeError # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2) # does not include WWW-Authenticate header, which breaks the standard. diff --git a/lib/api/applications.rb b/lib/api/applications.rb index b122cdefe4e..92717e04543 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # External applications API class Applications < Grape::API @@ -22,6 +24,22 @@ module API render_validation_error! application end end + + desc 'Get applications' do + success Entities::Application + end + get do + applications = ApplicationsFinder.new.execute + present applications, with: Entities::Application + end + + desc 'Delete an application' + delete ':id' do + application = ApplicationsFinder.new(params).execute + application.destroy + + status 204 + end end end end diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb new file mode 100644 index 00000000000..0f14d003065 --- /dev/null +++ b/lib/api/avatar.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module API + class Avatar < Grape::API + resource :avatar do + desc 'Return avatar url for a user' do + success Entities::Avatar + end + params do + requires :email, type: String, desc: 'Public email address of the user' + optional :size, type: Integer, desc: 'Single pixel dimension for Gravatar images' + end + get do + forbidden!('Unauthorized access') unless can?(current_user, :read_users_list) + + user = User.find_by_public_email(params[:email]) + user ||= User.new(email: params[:email]) + + present user, with: Entities::Avatar, size: params[:size] + end + end + end +end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index c3d93996816..a1851ba3627 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class AwardEmoji < Grape::API include PaginationParams @@ -12,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do AWARDABLES.each do |awardable_params| awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" @@ -100,9 +102,10 @@ module API end def can_award_awardable? - awardable.user_can_award?(current_user, params[:name]) + awardable.user_can_award?(current_user) end + # rubocop: disable CodeReuse/ActiveRecord def awardable @awardable ||= begin @@ -119,6 +122,7 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord def read_ability(awardable) case awardable diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 8ceffe9c5ef..ba554e00a16 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Badges < Grape::API include PaginationParams @@ -20,7 +22,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{source_type}" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Gets a list of #{source_type} badges viewable by the authenticated user." do detail 'This feature was introduced in GitLab 10.6.' success Entities::Badge diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 6c706b2b4e1..b7c77730afb 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Boards < Grape::API include BoardsResponses @@ -14,7 +16,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' @@ -33,6 +35,7 @@ module API success Entities::Board end get '/:board_id' do + authorize!(:read_board, user_project) present board, with: Entities::Board end end @@ -70,12 +73,10 @@ module API success Entities::List end params do - requires :label_id, type: Integer, desc: 'The ID of an existing label' + use :list_creation_params end post '/lists' do - unless available_labels_for(user_project).exists?(params[:label_id]) - render_api_error!({ error: 'Label not found!' }, 400) - end + authorize_list_type_resource! authorize!(:admin_list, user_project) diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index ead0943a74d..86d9b24802f 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module BoardsResponses extend ActiveSupport::Concern @@ -14,7 +16,7 @@ module API def create_list create_list_service = - ::Boards::Lists::CreateService.new(board_parent, current_user, { label_id: params[:label_id] }) + ::Boards::Lists::CreateService.new(board_parent, current_user, create_list_params) list = create_list_service.execute(board) @@ -25,6 +27,10 @@ module API end end + def create_list_params + params.slice(:label_id) + end + def move_list(list) move_list_service = ::Boards::Lists::MoveService.new(board_parent, current_user, { position: params[:position].to_i }) @@ -44,6 +50,18 @@ module API end end end + + # rubocop: disable CodeReuse/ActiveRecord + def authorize_list_type_resource! + unless available_labels_for(board_parent).exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + params :list_creation_params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 13cfba728fa..07f529b01bb 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -1,31 +1,26 @@ +# frozen_string_literal: true + require 'mime/types' module API class Branches < Grape::API include PaginationParams - BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) + BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) before { authorize! :download_code, user_project } helpers do - def find_branch!(branch_name) - begin - user_project.repository.find_branch(branch_name) || not_found!('Branch') - rescue Gitlab::Git::CommandError - render_api_error!('The branch refname is invalid', 400) - end - end - params :filter_params do optional :search, type: String, desc: 'Return list of branches matching the search criteria' + optional :sort, type: String, desc: 'Return list of branches sorted by the given field' end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository branches' do success Entities::Branch end @@ -39,12 +34,13 @@ module API repository = user_project.repository branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute - + branches = ::Kaminari.paginate_array(branches) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) present( - paginate(::Kaminari.paginate_array(branches)), + paginate(branches), with: Entities::Branch, + current_user: current_user, project: user_project, merged_branch_names: merged_branch_names ) @@ -63,7 +59,7 @@ module API get do branch = find_branch!(params[:branch]) - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end end @@ -75,10 +71,11 @@ module API success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/repository/branches/:branch/protect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project @@ -101,19 +98,21 @@ module API end if protected_branch.valid? - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end end + # rubocop: enable CodeReuse/ActiveRecord # Note: This API will be deprecated in favor of the protected branches API. desc 'Unprotect a single branch' do success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end + # rubocop: disable CodeReuse/ActiveRecord put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project @@ -121,15 +120,16 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create branch' do success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' - requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch', allow_blank: false end post ':id/repository/branches' do authorize_push_project @@ -140,6 +140,7 @@ module API if result[:status] == :success present result[:branch], with: Entities::Branch, + current_user: current_user, project: user_project else render_api_error!(result[:message], 400) @@ -148,7 +149,7 @@ module API desc 'Delete a branch' params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index d7138b2f2fe..19148758fc5 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class BroadcastMessages < Grape::API include PaginationParams diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb index c13154dc0ec..da756daadcc 100644 --- a/lib/api/circuit_breakers.rb +++ b/lib/api/circuit_breakers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class CircuitBreakers < Grape::API before { authenticated_as_admin! } @@ -11,37 +13,24 @@ module API end resource ':type' do namespace '', requirements: { type: 'repository_storage' } do - helpers do - def failing_storage_health - @failing_storage_health ||= Gitlab::Git::Storage::Health.for_failing_storages - end - - def storage_health - @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages - end - end - desc 'Get all git storages' do detail 'This feature was introduced in GitLab 9.5' - success Entities::RepositoryStorageHealth end get do - present storage_health, with: Entities::RepositoryStorageHealth + present [] end desc 'Get all failing git storages' do detail 'This feature was introduced in GitLab 9.5' - success Entities::RepositoryStorageHealth end get 'failing' do - present failing_storage_health, with: Entities::RepositoryStorageHealth + present [] end desc 'Reset all storage failures and open circuitbreaker' do detail 'This feature was introduced in GitLab 9.5' end delete do - Gitlab::Git::Storage::FailureInfo.reset_all! end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 829eef18795..08b4f8db8b0 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -5,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include PaginationParams before { authenticate! } @@ -21,12 +23,13 @@ module API optional :all, type: String, desc: 'Show all statuses, default: false' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) not_found!('Commit') unless user_project.commit(params[:sha]) - pipelines = user_project.pipelines.where(sha: params[:sha]) + pipelines = user_project.ci_pipelines.where(sha: params[:sha]) statuses = ::CommitStatus.where(pipeline: pipelines) statuses = statuses.latest unless to_boolean(params[:all]) statuses = statuses.where(ref: params[:ref]) if params[:ref].present? @@ -34,6 +37,7 @@ module API statuses = statuses.where(name: params[:name]) if params[:name].present? present paginate(statuses), with: Entities::CommitStatus end + # rubocop: enable CodeReuse/ActiveRecord desc 'Post status to a commit' do success Entities::CommitStatus @@ -49,6 +53,7 @@ module API optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :coverage, type: Float, desc: 'The total code coverage' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/statuses/:sha' do authorize! :create_commit_status, user_project @@ -70,7 +75,7 @@ module API pipeline = @project.pipeline_for(ref, commit.sha) unless pipeline - pipeline = @project.pipelines.create!( + pipeline = @project.ci_pipelines.create!( source: :external, sha: commit.sha, ref: ref, @@ -111,13 +116,14 @@ module API end MergeRequest.where(source_project: @project, source_branch: ref) - .update_all(head_pipeline_id: pipeline) if pipeline.latest? + .update_all(head_pipeline_id: pipeline.id) if pipeline.latest? present status, with: Entities::CommitStatus rescue StateMachines::InvalidTransition => e render_api_error!(e.message, 400) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 684955a1b24..9d23daafe95 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -6,28 +8,42 @@ module API before { authorize! :download_code, user_project } + helpers do + def user_access + @user_access ||= Gitlab::UserAccess.new(current_user, project: user_project) + end + + def authorize_push_to_branch!(branch) + unless user_access.can_push_to_branch?(branch) + forbidden!("You are not allowed to push into this branch") + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository commits' do success Entities::Commit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' - optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' - optional :path, type: String, desc: 'The file path' - optional :all, type: Boolean, desc: 'Every commit will be returned' + optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' + optional :path, type: String, desc: 'The file path' + optional :all, type: Boolean, desc: 'Every commit will be returned' + optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response' use :pagination end get ':id/repository/commits' do - path = params[:path] + path = params[:path] before = params[:until] - after = params[:since] - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] + after = params[:since] + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' unless params[:all] offset = (params[:page] - 1) * params[:per_page] - all = params[:all] + all = params[:all] + with_stats = params[:with_stats] commits = user_project.repository.commits(ref, path: path, @@ -47,7 +63,9 @@ module API paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present paginate(paginated_commits), with: Entities::Commit + serializer = with_stats ? Entities::CommitWithStats : Entities::Commit + + present paginate(paginated_commits), with: serializer end desc 'Commit multiple file changes as one commit' do @@ -55,15 +73,35 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + requires :actions, type: Array, desc: 'Actions to perform in commit' do + requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze + requires :file_path, type: String, desc: 'Full path to the file. Ex. `lib/class.rb`' + given action: ->(action) { action == 'move' } do + requires :previous_path, type: String, desc: 'Original full path to the file being moved. Ex. `lib/class1.rb`' + end + given action: ->(action) { %w[create move].include? action } do + optional :content, type: String, desc: 'File content' + end + given action: ->(action) { action == 'update' } do + requires :content, type: String, desc: 'File content' + end + optional :encoding, type: String, desc: '`text` or `base64`', default: 'text', values: %w[text base64] + given action: ->(action) { %w[update move delete].include? action } do + optional :last_commit_id, type: String, desc: 'Last known file commit id' + end + given action: ->(action) { action == 'chmod' } do + requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' + end + end optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end post ':id/repository/commits' do - authorize! :push_code, user_project + authorize_push_to_branch!(params[:branch]) attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) @@ -73,7 +111,10 @@ module API if result[:status] == :success commit_detail = user_project.repository.commit(result[:result]) - present commit_detail, with: Entities::CommitDetail + + Gitlab::WebIdeCommitsCounter.increment if find_user_from_warden + + present commit_detail, with: Entities::CommitDetail, stats: params[:stats] else render_api_error!(result[:message], 400) end @@ -120,6 +161,7 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) @@ -128,6 +170,7 @@ module API present paginate(notes), with: Entities::CommitNote end + # rubocop: enable CodeReuse/ActiveRecord desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' @@ -135,16 +178,49 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do - authorize! :push_code, user_project + authorize_push_to_branch!(params[:branch]) + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + find_branch!(params[:branch]) + + commit_params = { + commit: commit, + start_branch: params[:branch], + branch_name: params[:branch] + } + + result = ::Commits::CherryPickService + .new(user_project, current_user, commit_params) + .execute + + if result[:status] == :success + present user_project.repository.commit(result[:result]), + with: Entities::Commit + else + render_api_error!(result[:message], 400) + end + end + + desc 'Revert a commit in a branch' do + detail 'This feature was introduced in GitLab 11.5' + success Entities::Commit + end + params do + requires :sha, type: String, desc: 'Commit SHA to revert' + requires :branch, type: String, desc: 'Target branch name', allow_blank: false + end + post ':id/repository/commits/:sha/revert', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + authorize_push_to_branch!(params[:branch]) commit = user_project.commit(params[:sha]) not_found!('Commit') unless commit - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + find_branch!(params[:branch]) commit_params = { commit: commit, @@ -152,11 +228,13 @@ module API branch_name: params[:branch] } - result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + result = ::Commits::RevertService + .new(user_project, current_user, commit_params) + .execute if result[:status] == :success - branch = user_project.repository.find_branch(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit + present user_project.repository.commit(result[:result]), + with: Entities::Commit else render_api_error!(result[:message], 400) end diff --git a/lib/api/container_registry.rb b/lib/api/container_registry.rb new file mode 100644 index 00000000000..e4493910196 --- /dev/null +++ b/lib/api/container_registry.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module API + class ContainerRegistry < Grape::API + include PaginationParams + + REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } + before { authorize_read_container_images! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a project container repositories' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::Repository + end + params do + use :pagination + end + get ':id/registry/repositories' do + repositories = user_project.container_repositories.ordered + + present paginate(repositories), with: Entities::ContainerRegistry::Repository + end + + desc 'Delete repository' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + end + delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_admin_container_image! + + DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) + + status :accepted + end + + desc 'Get a list of repositories tags' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::Tag + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + use :pagination + end + get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_read_container_image! + + tags = Kaminari.paginate_array(repository.tags) + present paginate(tags), with: Entities::ContainerRegistry::Tag + end + + desc 'Delete repository tags (in bulk)' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all' + optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name' + optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month' + end + delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_admin_container_image! + + CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id, + declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord + + status :accepted + end + + desc 'Get a details about repository tag' do + detail 'This feature was introduced in GitLab 11.8.' + success Entities::ContainerRegistry::TagDetails + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :tag_name, type: String, desc: 'The name of the tag' + end + get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_read_container_image! + validate_tag! + + present tag, with: Entities::ContainerRegistry::TagDetails + end + + desc 'Delete repository tag' do + detail 'This feature was introduced in GitLab 11.8.' + end + params do + requires :repository_id, type: Integer, desc: 'The ID of the repository' + requires :tag_name, type: String, desc: 'The name of the tag' + end + delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do + authorize_destroy_container_image! + validate_tag! + + tag.delete + + status :ok + end + end + + helpers do + def authorize_read_container_images! + authorize! :read_container_image, user_project + end + + def authorize_read_container_image! + authorize! :read_container_image, repository + end + + def authorize_update_container_image! + authorize! :update_container_image, repository + end + + def authorize_destroy_container_image! + authorize! :admin_container_image, repository + end + + def authorize_admin_container_image! + authorize! :admin_container_image, repository + end + + def repository + @repository ||= user_project.container_repositories.find(params[:repository_id]) + end + + def tag + @tag ||= repository.tag(params[:tag_name]) + end + + def validate_tag! + not_found!('Tag') unless tag.valid? + end + end + end +end diff --git a/lib/api/custom_attributes_endpoints.rb b/lib/api/custom_attributes_endpoints.rb index 5000aa0d9ac..2149e04451e 100644 --- a/lib/api/custom_attributes_endpoints.rb +++ b/lib/api/custom_attributes_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module CustomAttributesEndpoints extend ActiveSupport::Concern @@ -30,6 +32,7 @@ module API params do use :custom_attributes_key end + # rubocop: disable CodeReuse/ActiveRecord get ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :read_custom_attribute @@ -38,12 +41,14 @@ module API present custom_attribute, with: Entities::CustomAttribute end + # rubocop: enable CodeReuse/ActiveRecord desc "Set a custom attribute on a #{attributable_name}" params do use :custom_attributes_key requires :value, type: String, desc: 'The value of the custom attribute' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :update_custom_attribute @@ -59,11 +64,13 @@ module API render_validation_error!(custom_attribute) end end + # rubocop: enable CodeReuse/ActiveRecord desc "Delete a custom attribute on a #{attributable_name}" params do use :custom_attributes_key end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/custom_attributes/:key' do resource = public_send(attributable_finder, params[:id]) # rubocop:disable GitlabSecurity/PublicSend authorize! :update_custom_attribute @@ -72,6 +79,7 @@ module API status 204 end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index b7aadc27e71..df6d2721977 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class DeployKeys < Grape::API include PaginationParams @@ -9,9 +11,11 @@ module API project.deploy_keys_projects.create(attrs) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_deploy_key(project, key_id) project.deploy_keys_projects.find_by!(deploy_key: key_id) end + # rubocop: enable CodeReuse/ActiveRecord end desc 'Return all deploy keys' @@ -27,7 +31,7 @@ module API params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } desc "Get a specific project's deploy keys" do @@ -36,11 +40,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/deploy_keys" do keys = user_project.deploy_keys_projects.preload(:deploy_key) present paginate(keys), with: Entities::DeployKeysProject end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get single deploy key' do success Entities::DeployKeysProject @@ -62,6 +68,7 @@ module API requires :title, type: String, desc: 'The name of the deploy key' optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end + # rubocop: disable CodeReuse/ActiveRecord post ":id/deploy_keys" do params[:key].strip! @@ -94,6 +101,7 @@ module API render_validation_error!(deploy_key_project) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing deploy key for a project' do success Entities::SSHKey @@ -112,9 +120,9 @@ module API can_push = params[:can_push].nil? ? deploy_keys_project.can_push : params[:can_push] title = params[:title] || deploy_keys_project.deploy_key.title - result = deploy_keys_project.update_attributes(can_push: can_push, - deploy_key_attributes: { id: params[:key_id], - title: title }) + result = deploy_keys_project.update(can_push: can_push, + deploy_key_attributes: { id: params[:key_id], + title: title }) if result present deploy_keys_project, with: Entities::DeployKeysProject @@ -147,12 +155,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/deploy_keys/:key_id" do deploy_key_project = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) not_found!('Deploy Key') unless deploy_key_project destroy_conditionally!(deploy_key_project) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 184fae0eb76..eb45df31ff9 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Deployments RESTful API endpoints class Deployments < Grape::API @@ -8,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment @@ -18,18 +20,20 @@ module API optional :order_by, type: String, values: %w[id iid created_at ref], default: 'id', desc: 'Return deployments ordered by `id` or `iid` or `created_at` or `ref`' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/deployments' do authorize! :read_deployment, user_project present paginate(user_project.deployments.order(params[:order_by] => params[:sort])), with: Entities::Deployment end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a specific deployment' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment end params do - requires :deployment_id, type: Integer, desc: 'The deployment ID' + requires :deployment_id, type: Integer, desc: 'The deployment ID' end get ':id/deployments/:deployment_id' do authorize! :read_deployment, user_project diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 13c34e3473a..91eb6a23701 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Discussions < Grape::API include PaginationParams @@ -15,7 +17,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a list of #{noteable_type.to_s.downcase} discussions" do success Entities::Discussion end @@ -23,6 +25,7 @@ module API requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/#{noteables_path}/:noteable_id/discussions" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) @@ -36,6 +39,7 @@ module API present paginate(discussions), with: Entities::Discussion end + # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} discussion" do success Entities::Discussion @@ -219,6 +223,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def readable_discussion_notes(noteable, discussion_id) notes = noteable.notes .where(discussion_id: discussion_id) @@ -228,6 +233,7 @@ module API notes.reject { |n| n.cross_reference_not_visible_for?(current_user) } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index c4537036a3a..9f1394571d8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Entities class WikiPageBasic < Grape::Entity @@ -10,6 +12,28 @@ module API expose :content end + class WikiAttachment < Grape::Entity + include Gitlab::FileMarkdownLinkBuilder + + expose :file_name + expose :file_path + expose :branch + expose :link do + expose :file_path, as: :url + expose :markdown do |_entity| + self.markdown_link + end + end + + def filename + object.file_name + end + + def secure_url + object.file_path + end + end + class UserSafe < Grape::Entity expose :id, :name, :username end @@ -30,8 +54,8 @@ module API end class User < UserBasic - expose :created_at - expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization + expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } + expose :bio, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization end class UserActivity < Grape::Entity @@ -55,12 +79,21 @@ module API expose :can_create_project?, as: :can_create_project expose :two_factor_enabled?, as: :two_factor_enabled expose :external + expose :private_profile end class UserWithAdmin < UserPublic expose :admin?, as: :is_admin end + class UserStatus < Grape::Entity + expose :emoji + expose :message + expose :message_html do |entity| + MarkupHelper.markdown_field(entity, :message) + end + end + class Email < Grape::Entity expose :id, :email end @@ -74,6 +107,7 @@ module API expose :project_id, :issues_events, :confidential_issues_events expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events expose :job_events + expose :push_events_branch_filter end class SharedGroup < Grape::Entity @@ -81,7 +115,11 @@ module API expose :group_name do |group_link, options| group_link.group.name end + expose :group_full_path do |group_link, options| + group_link.group.full_path + end expose :group_access, as: :group_access_level + expose :expires_at end class ProjectIdentity < Grape::Entity @@ -110,7 +148,9 @@ module API expose :import_status # TODO: Use `expose_nil` once we upgrade the grape-entity gem - expose :import_error, if: lambda { |status, _ops| status.import_error } + expose :import_error, if: lambda { |project, _ops| project.import_state&.last_error } do |project| + project.import_state.last_error + end end class BasicProjectDetails < ProjectIdentity @@ -125,21 +165,41 @@ module API # (fixed in https://github.com/rails/rails/pull/25976). project.tags.map(&:name).sort end + expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url + + expose :license_url, if: :license do |project| + license = project.repository.license_blob + + if license + Gitlab::Routing.url_helpers.project_blob_url(project, File.join(project.default_branch, license.path)) + end + end + + expose :license, with: 'API::Entities::LicenseBasic', if: :license do |project| + project.repository.license + end + expose :avatar_url do |project, options| project.avatar_url(only_path: false) end + expose :star_count, :forks_count expose :last_activity_at - + expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes - def self.preload_relation(projects_relation, options = {}) + # rubocop: disable CodeReuse/ActiveRecord + def self.preload_relation(projects_relation, options = {}) + # Preloading tags, should be done with using only `:tags`, + # as `:tags` are defined as: `has_many :tags, through: :taggings` + # N+1 is solved then by using `subject.tags.map(&:name)` + # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 projects_relation.preload(:project_feature, :route) - .preload(:import_state) - .preload(namespace: [:route, :owner], - tags: :taggings) + .preload(:import_state, :tags) + .preload(namespace: [:route, :owner]) end + # rubocop: enable CodeReuse/ActiveRecord end class Project < BasicProjectDetails @@ -191,10 +251,12 @@ module API expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace, using: 'API::Entities::NamespaceBasic' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda { |project, options| project.forked? } expose :import_status - expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } + + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| + project.import_state&.last_error + end expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -211,13 +273,19 @@ module API expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics - def self.preload_relation(projects_relation, options = {}) + # rubocop: disable CodeReuse/ActiveRecord + def self.preload_relation(projects_relation, options = {}) + # Preloading tags, should be done with using only `:tags`, + # as `:tags` are defined as: `has_many :tags, through: :taggings` + # N+1 is solved then by using `subject.tags.map(&:name)` + # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20555 super(projects_relation).preload(:group) - .preload(project_group_links: :group, + .preload(project_group_links: { group: :route }, fork_network: :root_project, - forked_project_link: :forked_from_project, - forked_from_project: [:route, :forks, namespace: :route, tags: :taggings]) + fork_network_member: :forked_from_project, + forked_from_project: [:route, :forks, :tags, namespace: :route]) end + # rubocop: enable CodeReuse/ActiveRecord def self.forks_counting_projects(projects_relation) projects_relation + projects_relation.map(&:forked_from_project).compact @@ -258,7 +326,7 @@ module API expose :request_access_enabled expose :full_name, :full_path - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? expose :parent_id end @@ -276,19 +344,23 @@ module API class GroupDetail < Group expose :projects, using: Entities::Project do |group, options| - GroupProjectsFinder.new( + projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], options: { only_owned: true } ).execute + + Entities::Project.prepare_relation(projects) end expose :shared_projects, using: Entities::Project do |group, options| - GroupProjectsFinder.new( + projects = GroupProjectsFinder.new( group: group, current_user: options[:current_user], options: { only_shared: true } ).execute + + Entities::Project.prepare_relation(projects) end end @@ -308,6 +380,10 @@ module API expose :additions, :deletions, :total end + class CommitWithStats < Commit + expose :stats, using: Entities::CommitStats + end + class CommitDetail < Commit expose :stats, using: Entities::CommitStats, if: :stats expose :status @@ -345,6 +421,14 @@ module API expose :developers_can_merge do |repo_branch, options| options[:project].protected_branches.developers_can?(:merge, repo_branch.name) end + + expose :can_push do |repo_branch, options| + Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name) + end + + expose :default do |repo_branch, options| + options[:project].default_branch == repo_branch.name + end end class TreeObject < Grape::Entity @@ -358,7 +442,7 @@ module API end class Snippet < Grape::Entity - expose :id, :title, :file_name, :description + expose :id, :title, :file_name, :description, :visibility expose :author, using: Entities::UserBasic expose :updated_at, :created_at expose :project_id @@ -404,6 +488,11 @@ module API expose :merge_access_levels, using: Entities::ProtectedRefAccess end + class ProtectedTag < Grape::Entity + expose :name + expose :create_access_levels, using: Entities::ProtectedRefAccess + end + class Milestone < Grape::Entity expose :id, :iid expose :project_id, if: -> (entity, options) { entity&.project_id } @@ -412,6 +501,10 @@ module API expose :state, :created_at, :updated_at expose :due_date expose :start_date + + expose :web_url do |milestone, _options| + Gitlab::UrlBuilder.build(milestone) + end end class IssueBasic < ProjectEntity @@ -497,10 +590,12 @@ module API expose :total_time_spent, as: :human_total_time_spent end + # rubocop: disable CodeReuse/ActiveRecord def total_time_spent # Avoids an N+1 query since timelogs are preloaded object.timelogs.map(&:time_spent).sum end + # rubocop: enable CodeReuse/ActiveRecord end class ExternalIssue < Grape::Entity @@ -510,6 +605,10 @@ module API class PipelineBasic < Grape::Entity expose :id, :sha, :ref, :status + + expose :web_url do |pipeline, _options| + Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) + end end class MergeRequestSimple < ProjectEntity @@ -520,6 +619,28 @@ module API end class MergeRequestBasic < ProjectEntity + expose :merged_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.merged_by + end + + expose :merged_at do |merge_request, _options| + merge_request.metrics&.merged_at + end + + expose :closed_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.latest_closed_by + end + + expose :closed_at do |merge_request, _options| + merge_request.metrics&.latest_closed_at + end + + expose :title_html, if: -> (_, options) { options[:render_html] } do |entity| + MarkupHelper.markdown_field(entity, :title) + end + expose :description_html, if: -> (_, options) { options[:render_html] } do |entity| + MarkupHelper.markdown_field(entity, :description) + end expose :target_branch, :source_branch expose :upvotes do |merge_request, options| if options[:issuable_metadata] @@ -559,7 +680,9 @@ module API expose :discussion_locked expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch - expose :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } + expose :allow_collaboration, if: -> (merge_request, _) { merge_request.for_fork? } + # Deprecated + expose :allow_collaboration, as: :allow_maintainer_to_push, if: -> (merge_request, _) { merge_request.for_fork? } expose :web_url do |merge_request, options| Gitlab::UrlBuilder.build(merge_request) @@ -581,22 +704,6 @@ module API merge_request.merge_request_diff.real_size end - expose :merged_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.merged_by - end - - expose :merged_at do |merge_request, _options| - merge_request.metrics&.merged_at - end - - expose :closed_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.latest_closed_by - end - - expose :closed_at do |merge_request, _options| - merge_request.metrics&.latest_closed_at - end - expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| merge_request.metrics&.latest_build_started_at end @@ -615,6 +722,12 @@ module API expose :diff_refs, using: Entities::DiffRefs + # Allow the status of a rebase to be determined + expose :merge_error + expose :rebase_in_progress?, as: :rebase_in_progress, if: -> (_, options) { options[:include_rebase_in_progress] } + + expose :diverged_commits_count, as: :diverged_commits_count, if: -> (_, options) { options[:include_diverged_commits_count] } + def build_available?(options) options[:project]&.feature_available?(:builds, options[:current_user]) end @@ -674,7 +787,7 @@ module API expose :system?, as: :system expose :noteable_id, :noteable_type - expose :position, if: ->(note, options) { note.diff_note? } do |note| + expose :position, if: ->(note, options) { note.is_a?(DiffNote) } do |note| note.position.to_h end @@ -692,6 +805,12 @@ module API expose :notes, using: Entities::Note end + class Avatar < Grape::Entity + expose :avatar_url do |avatarable, options| + avatarable.avatar_url(only_path: false, size: options[:size]) + end + end + class AwardEmoji < Grape::Entity expose :id expose :name @@ -749,28 +868,33 @@ module API class Todo < Grape::Entity expose :id - expose :project, using: Entities::BasicProjectDetails + expose :project, using: Entities::ProjectIdentity, if: -> (todo, _) { todo.project_id } + expose :group, using: 'API::Entities::NamespaceBasic', if: -> (todo, _) { todo.group_id } expose :author, using: Entities::UserBasic expose :action_name expose :target_type expose :target do |todo, options| - Entities.const_get(todo.target_type).represent(todo.target, options) + todo_target_class(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| target_type = todo.target_type.underscore - target_url = "namespace_project_#{target_type}_url" + target_url = "#{todo.parent.class.to_s.underscore}_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? Gitlab::Routing .url_helpers - .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend + .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body expose :state expose :created_at + + def todo_target_class(target_type) + ::API::Entities.const_get(target_type) + end end class NamespaceBasic < Grape::Entity @@ -805,7 +929,7 @@ module API class NotificationSetting < Grape::Entity expose :level expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do - ::NotificationSetting::EMAIL_EVENTS.each do |event| + ::NotificationSetting.email_events.each do |event| expose event end end @@ -844,12 +968,13 @@ module API if options[:group_members] options[:group_members].find { |member| member.source_id == project.namespace_id } else - project.group.group_member(options[:current_user]) + project.group.highest_group_member(options[:current_user]) end end end end + # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) @@ -874,6 +999,7 @@ module API relation end + # rubocop: enable CodeReuse/ActiveRecord end class LabelBasic < Grape::Entity @@ -968,11 +1094,42 @@ module API expose :password_authentication_enabled_for_web, as: :signin_enabled end - class Release < Grape::Entity + # deprecated old Release representation + class TagRelease < Grape::Entity expose :tag, as: :tag_name expose :description end + module Releases + class Link < Grape::Entity + expose :id + expose :name + expose :url + expose :external?, as: :external + end + + class Source < Grape::Entity + expose :format + expose :url + end + end + + class Release < TagRelease + expose :name + expose :description_html + expose :created_at + expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } + expose :commit, using: Entities::Commit + + expose :assets do + expose :assets_count, as: :count + expose :sources, using: Entities::Releases::Source + expose :links, using: Entities::Releases::Link do |release, options| + release.links.sorted + end + end + end + class Tag < Grape::Entity expose :name, :message, :target @@ -980,9 +1137,11 @@ module API options[:project].repository.commit(repo_tag.dereferenced_target) end - expose :release, using: Entities::Release do |repo_tag, options| + # rubocop: disable CodeReuse/ActiveRecord + expose :release, using: Entities::TagRelease do |repo_tag, options| options[:project].releases.find_by(tag: repo_tag.name) end + # rubocop: enable CodeReuse/ActiveRecord end class Runner < Grape::Entity @@ -990,7 +1149,7 @@ module API expose :description expose :ip_address expose :active - expose :is_shared + expose :instance_type?, as: :is_shared expose :name expose :online?, as: :online expose :status @@ -1004,7 +1163,8 @@ module API expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at - expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } + expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.instance_type? } + # rubocop: disable CodeReuse/ActiveRecord expose :projects, with: Entities::BasicProjectDetails do |runner, options| if options[:current_user].admin? runner.projects @@ -1012,6 +1172,8 @@ module API options[:current_user].authorized_projects.where(id: runner.projects) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord expose :groups, with: Entities::BasicGroupDetails do |runner, options| if options[:current_user].admin? runner.groups @@ -1019,6 +1181,7 @@ module API options[:current_user].authorized_groups.where(id: runner.groups) end end + # rubocop: enable CodeReuse/ActiveRecord end class RunnerRegistrationDetails < Grape::Entity @@ -1026,7 +1189,12 @@ module API end class JobArtifactFile < Grape::Entity - expose :filename, :size + expose :filename + expose :cached_size, as: :size + end + + class JobArtifact < Grape::Entity + expose :file_type, :size, :filename, :file_format end class JobBasic < Grape::Entity @@ -1036,10 +1204,16 @@ module API expose :user, with: User expose :commit, with: Commit expose :pipeline, with: PipelineBasic + + expose :web_url do |job, _options| + Gitlab::Routing.url_helpers.project_job_url(job.project, job) + end end class Job < JobBasic + # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5) expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } + expose :job_artifacts, as: :artifacts, using: JobArtifact expose :runner, with: Runner expose :artifacts_expire_at end @@ -1049,8 +1223,11 @@ module API end class Trigger < Grape::Entity + include ::API::Helpers::Presentable + expose :id - expose :token, :description + expose :token + expose :description expose :created_at, :updated_at, :last_used expose :owner, using: Entities::UserBasic end @@ -1096,11 +1273,14 @@ module API expose :deployable, using: Entities::Job end - class License < Grape::Entity + class LicenseBasic < Grape::Entity expose :key, :name, :nickname - expose :featured, as: :popular expose :url, as: :html_url expose(:source_url) { |license| license.meta['source'] } + end + + class License < LicenseBasic + expose :popular?, as: :popular expose(:description) { |license| license.meta['description'] } expose(:conditions) { |license| license.meta['conditions'] } expose(:permissions) { |license| license.meta['permissions'] } @@ -1109,6 +1289,7 @@ module API end class TemplatesList < Grape::Entity + expose :key expose :name end @@ -1133,7 +1314,11 @@ module API expose :token end - class ImpersonationToken < PersonalAccessTokenWithToken + class ImpersonationToken < PersonalAccessToken + expose :impersonation + end + + class ImpersonationTokenWithToken < PersonalAccessTokenWithToken expose :impersonation end @@ -1178,6 +1363,7 @@ module API class RunnerInfo < Grape::Entity expose :metadata_timeout, as: :timeout + expose :runner_session_url end class Step < Grape::Entity @@ -1193,7 +1379,13 @@ module API end class Artifacts < Grape::Entity - expose :name, :untracked, :paths, :when, :expire_in + expose :name + expose :untracked + expose :paths + expose :when + expose :expire_in + expose :artifact_type + expose :artifact_format end class Cache < Grape::Entity @@ -1244,12 +1436,6 @@ module API expose :submitted, as: :akismet_submitted end - class RepositoryStorageHealth < Grape::Entity - expose :storage_name - expose :failing_on_hosts - expose :total_failures - end - class CustomAttribute < Grape::Entity expose :key expose :value @@ -1298,7 +1484,9 @@ module API end class Application < Grape::Entity + expose :id expose :uid, as: :application_id + expose :name, as: :application_name expose :redirect_uri, as: :callback_url end @@ -1334,5 +1522,64 @@ module API badge.type == 'ProjectBadge' ? 'project' : 'group' end end + + class ResourceLabelEvent < Grape::Entity + expose :id + expose :user, using: Entities::UserBasic + expose :created_at + expose :resource_type do |event, options| + event.issuable.class.name + end + expose :resource_id do |event, options| + event.issuable.id + end + expose :label, using: Entities::LabelBasic + expose :action + end + + class Suggestion < Grape::Entity + expose :id + expose :from_original_line + expose :to_original_line + expose :from_line + expose :to_line + expose :appliable?, as: :appliable + expose :applied + expose :from_content + expose :to_content + end + + module Platform + class Kubernetes < Grape::Entity + expose :api_url + expose :namespace + expose :authorization_type + expose :ca_cert + end + end + + module Provider + class Gcp < Grape::Entity + expose :cluster_id + expose :status_name + expose :gcp_project_id + expose :zone + expose :machine_type + expose :num_nodes + expose :endpoint + end + end + + class Cluster < Grape::Entity + expose :id, :name, :created_at + expose :provider_type, :platform_type, :environment_scope, :cluster_type + expose :user, using: Entities::UserBasic + expose :platform_kubernetes, using: Entities::Platform::Kubernetes + expose :provider_gcp, using: Entities::Provider::Gcp + end + + class ClusterProject < Cluster + expose :project, using: Entities::BasicProjectDetails + end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb new file mode 100644 index 00000000000..00833ca7480 --- /dev/null +++ b/lib/api/entities/container_registry.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Entities + module ContainerRegistry + class Repository < Grape::Entity + expose :id + expose :name + expose :path + expose :location + expose :created_at + end + + class Tag < Grape::Entity + expose :name + expose :path + expose :location + end + + class TagDetails < Tag + expose :revision + expose :short_revision + expose :digest + expose :created_at + expose :total_size + end + end + end +end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 5c63ec028d9..0278c6c54a5 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Environments RESTfull API endpoints class Environments < Grape::API @@ -9,7 +11,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment @@ -72,7 +74,7 @@ module API success Entities::Environment end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The environment ID' end delete ':id/environments/:environment_id' do authorize! :update_environment, user_project @@ -86,12 +88,13 @@ module API success Entities::Environment end params do - requires :environment_id, type: Integer, desc: 'The environment ID' + requires :environment_id, type: Integer, desc: 'The environment ID' end post ':id/environments/:environment_id/stop' do - authorize! :create_deployment, user_project + authorize! :read_environment, user_project environment = user_project.environments.find(params[:environment_id]) + authorize! :stop_environment, environment environment.stop_with_action!(current_user) diff --git a/lib/api/events.rb b/lib/api/events.rb index b0713ff1d54..b98aa9f31e1 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -1,6 +1,9 @@ +# frozen_string_literal: true + module API class Events < Grape::API include PaginationParams + include APIGuard helpers do params :event_filter_params do @@ -16,13 +19,19 @@ module API end def present_events(events) - events = events.reorder(created_at: params[:sort]) + events = paginate(events) + + present events, with: Entities::Event + end - present paginate(events), with: Entities::Event + def find_events(source) + EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute end end resource :events do + allow_access_with_scope :read_user, if: -> (request) { request.get? } + desc "List currently authenticated user's events" do detail 'This feature was introduced in GitLab 9.3.' success Entities::Event @@ -32,10 +41,11 @@ module API use :event_filter_params use :sort_params end + get do authenticate! - events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) + events = find_events(current_user) present_events(events) end @@ -45,6 +55,8 @@ module API requires :id, type: String, desc: 'The ID or Username of the user' end resource :users do + allow_access_with_scope :read_user, if: -> (request) { request.get? } + desc 'Get the contribution events of a specified user' do detail 'This feature was introduced in GitLab 8.13.' success Entities::Event @@ -54,11 +66,12 @@ module API use :event_filter_params use :sort_params end + get ':id/events' do user = find_user(params[:id]) not_found!('User') unless user - events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) + events = find_events(user) present_events(events) end @@ -67,7 +80,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "List a Project's visible events" do success Entities::Event end @@ -76,8 +89,9 @@ module API use :event_filter_params use :sort_params end + get ":id/events" do - events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) + events = find_events(user_project) present_events(events) end diff --git a/lib/api/features.rb b/lib/api/features.rb index 11d848584d9..835aac05905 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Features < Grape::API before { authenticated_as_admin! } @@ -15,11 +17,11 @@ module API end def gate_targets(params) - targets = [] - targets << Feature.group(params[:feature_group]) if params[:feature_group] - targets << User.find_by_username(params[:user]) if params[:user] + Feature::Target.new(params).targets + end - targets + def gate_specified?(params) + Feature::Target.new(params).gate_specified? end end @@ -40,6 +42,7 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' + optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' end post ':name' do feature = Feature.get(params[:name]) @@ -48,13 +51,13 @@ module API case value when true - if targets.present? + if gate_specified?(params) targets.each { |target| feature.enable(target) } else feature.enable end when false - if targets.present? + if gate_specified?(params) targets.each { |target| feature.disable(target) } else feature.disable diff --git a/lib/api/files.rb b/lib/api/files.rb index 1598d3c00b8..ca59d330e1c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,10 +1,16 @@ +# frozen_string_literal: true + module API class Files < Grape::API - FILE_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) + include APIGuard + + FILE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } + helpers ::API::Helpers::HeadersHelpers + helpers do def commit_params(attrs) { @@ -40,10 +46,24 @@ module API } end + def blob_data + { + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, + encoding: "base64", + content_sha256: Digest::SHA256.hexdigest(@blob.data), + ref: params[:ref], + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) + } + end + params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' - requires :commit_message, type: String, desc: 'Commit message' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false + requires :commit_message, type: String, allow_blank: false, desc: 'Commit message' optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'The email of the author' optional :author_name, type: String, desc: 'The name of the author' @@ -61,36 +81,56 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do + allow_access_with_scope :read_repository, if: -> (request) { request.get? || request.head? } + + desc 'Get raw file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag commit' + requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! + set_http_headers(blob_data) + send_git_blob @repo, @blob end + desc 'Get file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + end + head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! - { - file_name: @blob.name, - file_path: @blob.path, - size: @blob.size, - encoding: "base64", - content: Base64.strict_encode64(@blob.data), - ref: params[:ref], - blob_id: @blob.id, - commit_id: @commit.id, - last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) - } + data = blob_data + + set_http_headers(data) + + data.merge(content: Base64.strict_encode64(@blob.data)) end desc 'Create new file in repository' diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index aa9fff25fc8..9a20ee8c8b9 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupBoards < Grape::API include BoardsResponses @@ -17,7 +19,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do desc 'Find a group board' do detail 'This feature was introduced in 10.6' @@ -70,12 +72,10 @@ module API success Entities::List end params do - requires :label_id, type: Integer, desc: 'The ID of an existing label' + use :list_creation_params end post '/lists' do - unless available_labels_for(board_parent).exists?(params[:label_id]) - render_api_error!({ error: 'Label not found!' }, 400) - end + authorize_list_type_resource! authorize!(:admin_list, user_group) diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 93fa0b95857..d4287e4a7c4 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupMilestones < Grape::API include MilestoneResponses @@ -10,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group milestones' do success Entities::Milestone end @@ -41,7 +43,7 @@ module API use :optional_params end post ":id/milestones" do - authorize! :admin_milestones, user_group + authorize! :admin_milestone, user_group create_milestone_for(user_group) end @@ -53,11 +55,21 @@ module API use :update_params end put ":id/milestones/:milestone_id" do - authorize! :admin_milestones, user_group + authorize! :admin_milestone, user_group update_milestone_for(user_group) end + desc 'Remove a project milestone' + delete ":id/milestones/:milestone_id" do + authorize! :admin_milestone, user_group + + milestone = user_group.milestones.find(params[:milestone_id]) + Milestones::DestroyService.new(user_group, current_user).execute(milestone) + + status(204) + end + desc 'Get all issues for a single group milestone' do success Entities::IssueBasic end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 55d5c7f1606..3f048e0dc56 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class GroupVariables < Grape::API include PaginationParams @@ -9,7 +11,7 @@ module API requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get group-level variables' do success Entities::Variable end @@ -27,6 +29,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do key = params[:key] variable = user_group.variables.find_by(key: key) @@ -35,6 +38,7 @@ module API present variable, with: Entities::Variable end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create a new variable in a group' do success Entities::Variable @@ -64,6 +68,7 @@ module API optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) @@ -77,6 +82,7 @@ module API render_validation_error!(variable) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing variable from a group' do success Entities::Variable @@ -84,12 +90,14 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do variable = user_group.variables.find_by(key: params[:key]) not_found!('GroupVariable') unless variable destroy_conditionally!(variable) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 03b6b30a0d8..64958ff982a 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Groups < Grape::API include PaginationParams @@ -32,13 +34,15 @@ module API optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' + optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Minimum access level of authenticated user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord def find_groups(params, parent_id = nil) - find_params = params.slice(:all_available, :custom_attributes, :owned) + find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level) find_params[:parent] = find_group!(parent_id) if parent_id find_params[:all_available] = find_params.fetch(:all_available, current_user&.full_private_access?) @@ -46,14 +50,29 @@ module API groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - groups = groups.reorder(params[:order_by] => params[:sort]) + order_options = { params[:order_by] => params[:sort] } + order_options["id"] ||= "asc" + groups = groups.reorder(order_options) groups end + # rubocop: enable CodeReuse/ActiveRecord def find_group_projects(params) group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + options = { + only_owned: !params[:with_shared], + include_subgroups: params[:include_subgroups] + } + + projects = GroupProjectsFinder.new( + group: group, + current_user: current_user, + params: project_finder_params, + options: options + ).execute + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = reorder_projects(projects) paginate(projects) end @@ -94,7 +113,7 @@ module API requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' - if ::Group.supports_nested_groups? + if ::Group.supports_nested_objects? optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' end @@ -121,7 +140,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end @@ -146,12 +165,13 @@ module API end params do use :with_custom_attributes + optional :with_projects, type: Boolean, default: true, desc: 'Omit project details' end get ":id" do group = find_group!(params[:id]) options = { - with: Entities::GroupDetail, + with: params[:with_projects] ? Entities::GroupDetail : Entities::Group, current_user: current_user } @@ -189,6 +209,10 @@ module API desc: 'Return only the ID, URL, name, and path of each project' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' + optional :with_shared, type: Boolean, default: true, desc: 'Include projects shared to this group' + optional :include_subgroups, type: Boolean, default: false, desc: 'Includes projects in subgroups of this group' use :pagination use :with_custom_attributes diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2ed331d4fd2..fa6c9777824 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers include Gitlab::Utils @@ -95,20 +97,20 @@ module API end def find_user(id) - if id =~ /^\d+$/ - User.find_by(id: id) - else - User.find_by(username: id) - end + UserFinder.new(id).find_by_id_or_username end + # rubocop: disable CodeReuse/ActiveRecord def find_project(id) + projects = Project.without_deleted + if id.is_a?(Integer) || id =~ /^\d+$/ - Project.find_by(id: id) + projects.find_by(id: id) elsif id.include?("/") - Project.find_by_full_path(id) + projects.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_project!(id) project = find_project(id) @@ -120,6 +122,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def find_group(id) if id.to_s =~ /^\d+$/ Group.find_by(id: id) @@ -127,6 +130,7 @@ module API Group.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_group!(id) group = find_group(id) @@ -138,6 +142,7 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ /^\d+$/ Namespace.find_by(id: id) @@ -145,6 +150,7 @@ module API Namespace.find_by_full_path(id) end end + # rubocop: enable CodeReuse/ActiveRecord def find_namespace!(id) namespace = find_namespace(id) @@ -156,6 +162,14 @@ module API end end + def find_branch!(branch_name) + if Gitlab::GitRefValidator.validate(branch_name) + user_project.repository.find_branch(branch_name) || not_found!('Branch') + else + render_api_error!('The branch refname is invalid', 400) + end + end + def find_project_label(id) labels = available_labels_for(user_project) label = labels.find_by_id(id) || labels.find_by_title(id) @@ -163,13 +177,17 @@ module API label || not_found!('Label') end + # rubocop: disable CodeReuse/ActiveRecord def find_project_issue(iid) IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def find_project_merge_request(iid) MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord def find_project_commit(id) user_project.commit_by(oid: id) @@ -180,11 +198,13 @@ module API SnippetsFinder.new(current_user, finder_params).find(id) end + # rubocop: disable CodeReuse/ActiveRecord def find_merge_request_with_access(iid, access_level = :read_merge_request) merge_request = user_project.merge_requests.find_by!(iid: iid) authorize! access_level, merge_request merge_request end + # rubocop: enable CodeReuse/ActiveRecord def find_build!(id) user_project.builds.find(id.to_i) @@ -215,8 +235,8 @@ module API forbidden! unless current_user.admin? end - def authorize!(action, subject = :global) - forbidden! unless can?(current_user, action, subject) + def authorize!(action, subject = :global, reason = nil) + forbidden!(reason) unless can?(current_user, action, subject) end def authorize_push_project @@ -272,12 +292,15 @@ module API attrs[key] = params_hash[key] end end - ActionController::Parameters.new(attrs).permit! + permitted_attrs = ActionController::Parameters.new(attrs).permit! + permitted_attrs.to_h end + # rubocop: disable CodeReuse/ActiveRecord def filter_by_iid(items, iid) items.where(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord def filter_by_search(items, text) items.search(text) @@ -347,18 +370,19 @@ module API end def handle_api_exception(exception) - if sentry_enabled? && report_exception?(exception) + if report_exception?(exception) define_params_for_grape_middleware - sentry_context - Raven.capture_exception(exception, extra: params) + Gitlab::Sentry.context(current_user) + Gitlab::Sentry.track_acceptable_exception(exception, extra: params) end # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 trace = exception.backtrace - message = "\n#{exception.class} (#{exception.message}):\n" + message = ["\n#{exception.class} (#{exception.message}):\n"] message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) message << " " << trace.join("\n ") + message = message.join API.logger.add Logger::FATAL, message @@ -374,20 +398,23 @@ module API # project helpers + # rubocop: disable CodeReuse/ActiveRecord def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end + # rubocop: enable CodeReuse/ActiveRecord def project_finder_params - finder_params = {} + finder_params = { without_deleted: true } finder_params[:owned] = true if params[:owned].present? finder_params[:non_public] = true if params[:membership].present? finder_params[:starred] = true if params[:starred].present? finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] - finder_params[:archived] = params[:archived] + finder_params[:archived] = archived_param unless params[:archived].nil? finder_params[:search] = params[:search] if params[:search] finder_params[:user] = params.delete(:user) if params[:user] finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] + finder_params[:min_access_level] = params[:min_access_level] if params[:min_access_level] finder_params end @@ -469,6 +496,11 @@ module API def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' + header['Content-Disposition'] = content_disposition('inline', blob.name) + + # Let Workhorse examine the content and determine the better content disposition + header[Gitlab::Workhorse::DETECT_HEADER] = "true" + header(*Gitlab::Workhorse.send_git_blob(repository, blob)) end @@ -484,7 +516,7 @@ module API # `request`. We workaround this by defining methods that returns the right # values. def define_params_for_grape_middleware - self.define_singleton_method(:request) { Rack::Request.new(env) } + self.define_singleton_method(:request) { ActionDispatch::Request.new(env) } self.define_singleton_method(:params) { request.params.symbolize_keys } end @@ -495,5 +527,17 @@ module API exception.status == 500 end + + def archived_param + return 'only' if params[:archived] + + params[:archived] + end + + def content_disposition(disposition, filename) + disposition += %(; filename=#{filename.inspect}) if filename.present? + + disposition + end end end diff --git a/lib/api/helpers/badges_helpers.rb b/lib/api/helpers/badges_helpers.rb index 1f8afbf3c90..46ce5b4e7b5 100644 --- a/lib/api/helpers/badges_helpers.rb +++ b/lib/api/helpers/badges_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module BadgesHelpers diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 9993caa5249..7551ca50a7f 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CommonHelpers diff --git a/lib/api/helpers/custom_attributes.rb b/lib/api/helpers/custom_attributes.rb index 10d652e33f5..88208226c40 100644 --- a/lib/api/helpers/custom_attributes.rb +++ b/lib/api/helpers/custom_attributes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomAttributes @@ -12,6 +14,7 @@ module API desc: 'Filter with custom attributes' end + # rubocop: disable CodeReuse/ActiveRecord def with_custom_attributes(collection_or_resource, options = {}) options = options.merge( with_custom_attributes: params[:with_custom_attributes] && @@ -24,6 +27,7 @@ module API [collection_or_resource, options] end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index dd4f6c41131..1058f4e8a5e 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module CustomValidators @@ -8,8 +10,21 @@ module API raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end + + class IntegerNoneAny < Grape::Validations::Base + def validate_param!(attr_name, params) + value = params[attr_name] + + return if value.is_a?(Integer) || + [IssuableFinder::FILTER_NONE, IssuableFinder::FILTER_ANY].include?(value.to_s.downcase) + + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be an integer, 'None' or 'Any'" + end + end end end end Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) +Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny) diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb new file mode 100644 index 00000000000..7553af9d156 --- /dev/null +++ b/lib/api/helpers/headers_helpers.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Helpers + module HeadersHelpers + def set_http_headers(header_data) + header_data.each do |key, value| + if value.is_a?(Enumerable) + raise ArgumentError.new("Header value should be a string") + end + + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value.to_s + end + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 83151be82ad..4eaaca96b49 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module InternalHelpers diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index a50ea0b52aa..73d58ee7f37 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable GitlabSecurity/PublicSend module API @@ -10,6 +12,35 @@ module API def authorize_admin_source!(source_type, source) authorize! :"admin_#{source_type}", source end + + def find_all_members(source_type, source) + members = source_type == 'project' ? find_all_members_for_project(source) : find_all_members_for_group(source) + members.non_invite + .non_request + end + + # rubocop: disable CodeReuse/ActiveRecord + def find_all_members_for_project(project) + shared_group_ids = project.project_group_links.pluck(:group_id) + project_group_ids = project.group&.self_and_ancestors&.pluck(:id) + source_ids = [project.id, project_group_ids, shared_group_ids] + .flatten + .compact + Member.includes(:user) + .joins(user: :project_authorizations) + .where(project_authorizations: { project_id: project.id }) + .where(source_id: source_ids) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def find_all_members_for_group(group) + source_ids = group.self_and_ancestors.pluck(:id) + Member.includes(:user) + .where(source_id: source_ids) + .where(source_type: 'Namespace') + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index b4bfb677d72..216b2c45741 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module NotesHelpers @@ -92,10 +94,9 @@ module API parent = noteable_parent(noteable) - if opts[:created_at] - opts.delete(:created_at) unless - current_user.admin? || parent.owned_by?(current_user) - end + opts.delete(:created_at) unless current_user.can?(:set_note_created_at, policy_object) + + opts[:updated_at] = opts[:created_at] if opts[:created_at] project = parent if parent.is_a?(Project) ::Notes::CreateService.new(project, current_user, opts).execute diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 3308212216e..de59c915d66 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Pagination @@ -91,6 +93,7 @@ module API @request_context = request_context end + # rubocop: disable CodeReuse/ActiveRecord def paginate(relation) pagination = KeysetPaginationInfo.new(relation, request_context) @@ -112,6 +115,7 @@ module API paged_relation end + # rubocop: enable CodeReuse/ActiveRecord private @@ -142,7 +146,7 @@ module API end def add_default_pagination_headers - header 'X-Per-Page', per_page.to_s + header 'X-Per-Page', per_page.to_s end def add_navigation_links(next_page_params) @@ -174,15 +178,27 @@ module API end def paginate(relation) - relation = add_default_order(relation) - - relation.page(params[:page]).per(params[:per_page]).tap do |data| + paginate_with_limit_optimization(add_default_order(relation)).tap do |data| add_pagination_headers(data) end end private + def paginate_with_limit_optimization(relation) + pagination_data = relation.page(params[:page]).per(params[:per_page]) + return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + + limited_total_count = pagination_data.total_count_with_limit + if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT + pagination_data.without_count + else + pagination_data + end + end + + # rubocop: disable CodeReuse/ActiveRecord def add_default_order(relation) if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) @@ -190,6 +206,7 @@ module API relation end + # rubocop: enable CodeReuse/ActiveRecord def add_pagination_headers(paginated_data) header 'X-Per-Page', paginated_data.limit_value.to_s diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb new file mode 100644 index 00000000000..973c2132efe --- /dev/null +++ b/lib/api/helpers/presentable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Helpers + ## + # This module makes it possible to use `app/presenters` with + # Grape Entities. It instantiates model presenter and passes + # options defined in the API endpoint to the presenter itself. + # + # present object, with: Entities::Something, + # current_user: current_user, + # another_option: 'my options' + # + # Example above will make `current_user` and `another_option` + # values available in the subclass of `Gitlab::View::Presenter` + # thorough a separate method in the presenter. + # + # The model class needs to have `::Presentable` module mixed in + # if you want to use `API::Helpers::Presentable`. + # + module Presentable + extend ActiveSupport::Concern + + def initialize(object, options = {}) + super(object.present(options), options) + end + end + end +end diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb index 94798a8cb51..1b5dc281e38 100644 --- a/lib/api/helpers/project_snapshots_helpers.rb +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectSnapshotsHelpers diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 381d5e8968c..e6a72b949f9 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module ProjectsHelpers @@ -26,6 +28,7 @@ module API optional :avatar, type: File, desc: 'Avatar image for project' optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" end params :optional_project_params do diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index bc7333ca4b3..793ae11b41d 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module RelatedResourcesHelpers diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 35ac0b4cbca..16df8e830e1 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module Helpers module Runner @@ -24,7 +26,7 @@ module API end def get_runner_ip - { ip_address: request.ip } + { ip_address: request.env["HTTP_X_FORWARDED_FOR"] || request.ip } end def current_runner @@ -59,6 +61,11 @@ module API def max_artifacts_size Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i end + + def job_forbidden!(job, reason) + header 'Job-Status', job.status + forbidden!(reason) + end end end end diff --git a/lib/api/helpers/version.rb b/lib/api/helpers/version.rb new file mode 100644 index 00000000000..7f53094e90c --- /dev/null +++ b/lib/api/helpers/version.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + module Helpers + class Version + include Helpers::RelatedResourcesHelpers + + def initialize(version) + @version = version.to_s + + unless API.versions.include?(version) + raise ArgumentError, 'Unknown API version!' + end + end + + def root_path + File.join('/', API.prefix.to_s, @version) + end + + def root_url + @root_url ||= expose_url(root_path) + end + + def to_s + @version + end + end + end +end diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb new file mode 100644 index 00000000000..bb4e536cf57 --- /dev/null +++ b/lib/api/import_github.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module API + class ImportGithub < Grape::API + rescue_from Octokit::Unauthorized, with: :provider_unauthorized + + helpers do + def client + @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options) + end + + def access_params + { github_access_token: params[:personal_access_token] } + end + + def client_options + {} + end + + def provider + :github + end + end + + desc 'Import a GitHub project' do + detail 'This feature was introduced in GitLab 11.3.4.' + success Entities::ProjectEntity + end + params do + requires :personal_access_token, type: String, desc: 'GitHub personal access token' + requires :repo_id, type: Integer, desc: 'GitHub repository ID' + optional :new_name, type: String, desc: 'New repo name' + requires :target_namespace, type: String, desc: 'Namespace to import repo into' + end + post 'import/github' do + result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) + + if result[:status] == :success + present ProjectSerializer.new.represent(result[:project]) + else + status result[:http_status] + { errors: result[:message] } + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index a9803be9f69..9488b3469d9 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Internal access API class Internal < Grape::API @@ -6,19 +8,28 @@ module API helpers ::API::Helpers::InternalHelpers helpers ::Gitlab::Identifier + UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze + + helpers do + def response_with_status(code: 200, success: true, message: nil, **extra_options) + status code + { status: success, message: message }.merge(extra_options).compact + end + end + namespace 'internal' do - # Check if git command is allowed to project + # Check if git command is allowed for project # # Params: # key_id - ssh key id for Git over SSH - # user_id - user id for Git over HTTP + # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode + # username - user name for Git over SSH in keyless SSH cert mode # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList + # rubocop: disable CodeReuse/ActiveRecord post "/allowed" do - status 200 - # Stores some Git-specific env thread-safely env = parse_env Gitlab::Git::HookEnv.set(gl_repository, env) if project @@ -28,6 +39,8 @@ module API Key.find_by(id: params[:key_id]) elsif params[:user_id] User.find_by(id: params[:user_id]) + elsif params[:username] + UserFinder.new(params[:username]).find_by_username end protocol = params[:protocol] @@ -46,35 +59,65 @@ module API namespace_path: namespace_path, project_path: project_path, redirected_path: redirected_path) - begin - access_checker.check(params[:action], params[:changes]) - @project ||= access_checker.project - rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e - break { status: false, message: e.message } - end + check_result = begin + result = access_checker.check(params[:action], params[:changes]) + @project ||= access_checker.project + result + rescue Gitlab::GitAccess::UnauthorizedError => e + break response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::TimeoutError => e + break response_with_status(code: 503, success: false, message: e.message) + rescue Gitlab::GitAccess::NotFoundError => e + break response_with_status(code: 404, success: false, message: e.message) + end log_user_activity(actor) - { - status: true, - gl_repository: gl_repository, - gl_username: user&.username, - - # This repository_path is a bogus value but gitlab-shell still requires - # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 - repository_path: '/', + case check_result + when ::Gitlab::GitAccessResult::Success + payload = { + gl_repository: gl_repository, + gl_id: Gitlab::GlId.gl_id(user), + gl_username: user&.username, + git_config_options: [], + + # This repository_path is a bogus value but gitlab-shell still requires + # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 + repository_path: '/', + + gitaly: gitaly_payload(params[:action]) + } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end - gitaly: gitaly_payload(params[:action]) - } + response_with_status(**payload) + when ::Gitlab::GitAccessResult::CustomAction + response_with_status(code: 300, message: check_result.message, payload: check_result.payload) + else + response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR) + end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord post "/lfs_authenticate" do status 200 - key = Key.find(params[:key_id]) - key.update_last_used_at + if params[:key_id] + actor = Key.find(params[:key_id]) + actor.update_last_used_at + elsif params[:user_id] + actor = User.find_by(id: params[:user_id]) + raise ActiveRecord::RecordNotFound.new("No such user id!") unless actor + else + raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") + end - token_handler = Gitlab::LfsToken.new(key) + token_handler = Gitlab::LfsToken.new(actor) { username: token_handler.actor_name, @@ -82,6 +125,7 @@ module API repository_http_path: project.http_url_to_repo } end + # rubocop: enable CodeReuse/ActiveRecord get "/merge_request_urls" do merge_request_urls @@ -90,6 +134,7 @@ module API # # Get a ssh key using the fingerprint # + # rubocop: disable CodeReuse/ActiveRecord get "/authorized_keys" do fingerprint = params.fetch(:fingerprint) do Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint @@ -98,20 +143,25 @@ module API not_found!("Key") if key.nil? present key, with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord # - # Discover user by ssh key or user id + # Discover user by ssh key, user id or username # + # rubocop: disable CodeReuse/ActiveRecord get "/discover" do if params[:key_id] key = Key.find(params[:key_id]) user = key.user elsif params[:user_id] user = User.find_by(id: params[:user_id]) + elsif params[:username] + user = UserFinder.new(params[:username]).find_by_username end present user, with: Entities::UserSafe end + # rubocop: enable CodeReuse/ActiveRecord get "/check" do { @@ -138,25 +188,34 @@ module API end end + # rubocop: disable CodeReuse/ActiveRecord post '/two_factor_recovery_codes' do status 200 - key = Key.find_by(id: params[:key_id]) + if params[:key_id] + key = Key.find_by(id: params[:key_id]) - if key - key.update_last_used_at - else - break { 'success' => false, 'message' => 'Could not find the given key' } - end + if key + key.update_last_used_at + else + break { 'success' => false, 'message' => 'Could not find the given key' } + end - if key.is_a?(DeployKey) - break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } - end + if key.is_a?(DeployKey) + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user - user = key.user + unless user + break { success: false, message: 'Could not find a user for the given key' } + end + elsif params[:user_id] + user = User.find_by(id: params[:user_id]) - unless user - break { success: false, message: 'Could not find a user for the given key' } + unless user + break { success: false, message: 'Could not find the given user' } + end end unless user.two_factor_enabled? @@ -171,6 +230,7 @@ module API { success: true, recovery_codes: codes } end + # rubocop: enable CodeReuse/ActiveRecord post '/pre_receive' do status 200 @@ -196,8 +256,9 @@ module API post '/post_receive' do status 200 + PostReceive.perform_async(params[:gl_repository], params[:identifier], - params[:changes]) + params[:changes], params[:push_options].to_a) broadcast_message = BroadcastMessage.current&.last&.message reference_counter_decreased = Gitlab::ReferenceCounter.new(params[:gl_repository]).decrease diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b64f465ce56..afa3ac80121 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Issues < Grape::API include PaginationParams @@ -6,7 +8,17 @@ module API helpers ::Gitlab::IssuableMetadata + # EE::API::Issues would override the following helpers + helpers do + params :issues_params_ee do + end + + params :issue_params_ee do + end + end + helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_issues(args = {}) args = declared_params.merge(args) @@ -16,10 +28,11 @@ module API args[:scope] = args[:scope].underscore if args[:scope] issues = IssuesFinder.new(current_user, args).execute - .preload(:assignees, :labels, :notes, :timelogs, :project) + .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by) issues.reorder(args[:order_by] => args[:sort]) end + # rubocop: enable CodeReuse/ActiveRecord params :issues_params do optional :labels, type: String, desc: 'Comma-separated list of label names' @@ -36,14 +49,17 @@ module API optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination + + use :issues_params_ee end - params :issue_params_ce do + params :issue_params do optional :description, type: String, desc: 'The description of an issue' optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' @@ -52,10 +68,8 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - end - params :issue_params do - use :issue_params_ce + use :issue_params_ee end end @@ -87,7 +101,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group issues' do success Entities::IssueBasic end @@ -114,7 +128,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints desc 'Get a list of project issues' do @@ -162,6 +176,9 @@ module API desc: 'The IID of a merge request for which to resolve discussions' optional :discussion_to_resolve, type: String, desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`' + optional :iid, type: Integer, + desc: 'The internal ID of a project issue. Available only for admins and project owners.' + use :issue_params end post ':id/issues' do @@ -169,10 +186,8 @@ module API authorize! :create_issue, user_project - # Setting created_at time only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user - params.delete(:created_at) - end + params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project) + params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project) issue_params = declared_params(include_missing: false) @@ -206,14 +221,15 @@ module API at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :discussion_locked, :labels, :created_at, :due_date, :confidential, :state_event end + # rubocop: disable CodeReuse/ActiveRecord put ':id/issues/:issue_iid' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322') issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - # Setting created_at time only allowed for admins and project owners - unless current_user.admin? || user_project.owner == current_user + # Setting created_at time only allowed for admins and project/group owners + unless current_user.admin? || user_project.owner == current_user || current_user.owned_groups.include?(user_project.owner) params.delete(:updated_at) end @@ -233,6 +249,7 @@ module API render_validation_error!(issue) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Move an existing issue' do success Entities::Issue @@ -241,6 +258,7 @@ module API requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' requires :to_project_id, type: Integer, desc: 'The ID of the new project' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/issues/:issue_iid/move' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323') @@ -257,11 +275,13 @@ module API render_api_error!(error.message, 400) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a project issue' params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/issues/:issue_iid" do issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue @@ -272,13 +292,39 @@ module API Issuable::DestroyService.new(user_project, current_user).execute(issue) end end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'List merge requests that are related to the issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/related_merge_requests' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + .execute(issue) + .flatten + .map(&:iid) + + merge_requests = + if merge_request_iids.present? + MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute + else + MergeRequest.none + end + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end - desc 'List merge requests closing issue' do + desc 'List merge requests closing issue' do success Entities::MergeRequestBasic end params do requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/issues/:issue_iid/closed_by' do issue = find_project_issue(params[:issue_iid]) @@ -287,8 +333,9 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord - desc 'List participants for an issue' do + desc 'List participants for an issue' do success Entities::UserBasic end params do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 32379d7c8ab..933bd067e26 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class JobArtifacts < Grape::API before { authenticate_non_get! } @@ -12,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.10' end @@ -25,12 +27,34 @@ module API requirements: { ref_name: /.+/ } do authorize_download_artifacts! - builds = user_project.latest_successful_builds_for(params[:ref_name]) - latest_build = builds.find_by!(name: params[:job]) + latest_build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) present_carrierwave_file!(latest_build.artifacts_file) end + desc 'Download a specific file from artifacts archive from a ref' do + detail 'This feature was introduced in GitLab 11.5' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + get ':id/jobs/artifacts/:ref_name/raw/*artifact_path', + format: false, + requirements: { ref_name: /.+/ } do + authorize_download_artifacts! + + build = user_project.latest_successful_build_for!(params[:job], params[:ref_name]) + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + + bad_request! unless path.valid? + + send_artifacts_entry(build, path) + end + desc 'Download the artifacts archive from a job' do detail 'This feature was introduced in GitLab 8.5' end @@ -61,6 +85,7 @@ module API path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) + bad_request! unless path.valid? send_artifacts_entry(build, path) diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 54d1acbd412..59f0dbe8a9b 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Jobs < Grape::API include PaginationParams @@ -7,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', @@ -34,29 +36,39 @@ module API use :optional_scope use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/jobs' do + authorize_read_builds! + builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project) + builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, pipeline: :project) present paginate(builds), with: Entities::Job end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get pipeline jobs' do success Entities::Job end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' use :optional_scope use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/pipelines/:pipeline_id/jobs' do - pipeline = user_project.pipelines.find(params[:pipeline_id]) + authorize!(:read_pipeline, user_project) + pipeline = user_project.ci_pipelines.find(params[:pipeline_id]) + authorize!(:read_build, pipeline) + builds = pipeline.builds builds = filter_builds(builds, params[:scope]) + builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace]) present paginate(builds), with: Entities::Job end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a specific job of a project' do success Entities::Job @@ -144,7 +156,7 @@ module API present build, with: Entities::Job end - desc 'Trigger a manual job' do + desc 'Trigger a actionable job (manual, delayed, etc)' do success Entities::Job detail 'This feature was added in GitLab 8.11' end @@ -167,6 +179,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? @@ -177,6 +190,7 @@ module API builds.where(status: available_statuses && scope) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 767f27ef334..d5280a0035d 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Keys API class Keys < Grape::API @@ -12,7 +14,7 @@ module API key = Key.find(params[:id]) - present key, with: Entities::SSHKeyWithUser + present key, with: Entities::SSHKeyWithUser, current_user: current_user end end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 81eaf56e48e..d5eb2b94669 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Labels < Grape::API include PaginationParams @@ -7,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::Label end @@ -27,6 +29,7 @@ module API optional :description, type: String, desc: 'The description of label to be created' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end + # rubocop: disable CodeReuse/ActiveRecord post ':id/labels' do authorize! :admin_label, user_project @@ -43,6 +46,7 @@ module API render_validation_error!(label) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing label' do success Entities::Label @@ -50,6 +54,7 @@ module API params do requires :name, type: String, desc: 'The name of the label to be deleted' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/labels' do authorize! :admin_label, user_project @@ -58,18 +63,20 @@ module API destroy_conditionally!(label) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Update an existing label. At least one optional parameter is required.' do success Entities::Label end params do - requires :name, type: String, desc: 'The name of the label to be updated' + requires :name, type: String, desc: 'The name of the label to be updated' optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The new description of label' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true at_least_one_of :new_name, :color, :description, :priority end + # rubocop: disable CodeReuse/ActiveRecord put ':id/labels' do authorize! :admin_label, user_project @@ -95,6 +102,7 @@ module API present label, with: Entities::Label, current_user: current_user, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index d202eaa4c49..a7672021db0 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Lint < Grape::API namespace :ci do @@ -6,7 +8,8 @@ module API requires :content, type: String, desc: 'Content of .gitlab-ci.yml' end post '/lint' do - error = Gitlab::Ci::YamlProcessor.validation_message(params[:content]) + error = Gitlab::Ci::YamlProcessor.validation_message(params[:content], + user: current_user) status 200 diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index b9ed68aa584..de77bef43ce 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Markdown < Grape::API params do @@ -10,9 +12,8 @@ module API detail "This feature was introduced in GitLab 11.0." end post do - # Explicitly set CommonMark as markdown engine to use. - # Remove this set when https://gitlab.com/gitlab-org/gitlab-ce/issues/43011 is done. - context = { markdown_engine: :common_mark, only_path: false } + context = { only_path: false, current_user: current_user } + context[:pipeline] = params[:gfm] ? :full : :plain_markdown if params[:project] project = Project.find_by_full_path(params[:project]) @@ -24,9 +25,7 @@ module API context[:skip_project_check] = true end - context[:pipeline] = params[:gfm] ? :full : :plain_markdown - - { html: Banzai.render(params[:text], context) } + { html: Banzai.render_and_post_process(params[:text], context) } end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index 8b12986d09e..461ffe71a62 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Members < Grape::API include PaginationParams @@ -10,7 +12,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end @@ -18,6 +20,7 @@ module API optional :query, type: String, desc: 'A query string to search for members' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/members" do source = find_source(source_type, params[:id]) @@ -27,6 +30,26 @@ module API present members, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Gets a list of group or project members viewable by the authenticated user, including those who gained membership through ancestor group.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + use :pagination + end + # rubocop: disable CodeReuse/ActiveRecord + get ":id/members/all" do + source = find_source(source_type, params[:id]) + + members = find_all_members(source_type, source) + members = members.includes(:user).references(:user).merge(User.search(params[:query])) if params[:query].present? + members = paginate(members) + + present members, with: Entities::Member + end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a member of a group or project.' do success Entities::Member @@ -34,6 +57,7 @@ module API params do requires :user_id, type: Integer, desc: 'The user ID of the member' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -42,6 +66,7 @@ module API present member, with: Entities::Member end + # rubocop: enable CodeReuse/ActiveRecord desc 'Adds a member to a group or project.' do success Entities::Member @@ -51,6 +76,7 @@ module API requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) @@ -58,7 +84,10 @@ module API member = source.members.find_by(user_id: params[:user_id]) conflict!('Member already exists') if member - member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + user = User.find_by_id(params[:user_id]) + not_found!('User') unless user + + member = source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) if !member not_allowed! # This currently can only be reached in EE @@ -68,6 +97,7 @@ module API render_validation_error!(member) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Updates a member of a group or project.' do success Entities::Member @@ -77,6 +107,7 @@ module API requires :access_level, type: Integer, desc: 'A valid access level' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end + # rubocop: disable CodeReuse/ActiveRecord put ":id/members/:user_id" do source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) @@ -93,11 +124,13 @@ module API render_validation_error!(updated_member) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Removes a user from a group or project.' params do requires :user_id, type: Integer, desc: 'The user ID of the member' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) member = source.members.find_by!(user_id: params[:user_id]) @@ -106,6 +139,7 @@ module API ::Members::DestroyService.new(current_user).execute(member) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 95ef8f42954..6ad30aa56e0 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # MergeRequestDiff API class MergeRequestDiffs < Grape::API @@ -8,7 +10,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index b1e510d72de..132b19164d0 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class MergeRequests < Grape::API include PaginationParams @@ -28,9 +30,9 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_merge_requests(args = {}) args = declared_params.merge(args) - args[:milestone_title] = args.delete(:milestone) args[:label_name] = args.delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] @@ -38,13 +40,14 @@ module API merge_requests = MergeRequestsFinder.new(current_user, args).execute .reorder(args[:order_by] => args[:sort]) merge_requests = paginate(merge_requests) - .preload(:target_project) + .preload(:source_project, :target_project) return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs, metrics: [:latest_closed_by, :merged_by]) end + # rubocop: enable CodeReuse/ActiveRecord def merge_request_pipelines_with_access authorize! :read_pipeline, user_project @@ -71,9 +74,22 @@ module API options end + def authorize_push_to_merge_request!(merge_request) + forbidden!('Source branch does not exist') unless + merge_request.source_branch_exists? + + user_access = Gitlab::UserAccess.new( + current_user, + project: merge_request.source_project + ) + + forbidden!('Cannot push to source branch') unless + user_access.can_push_to_branch?(merge_request.source_branch) + end + params :merge_requests_params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' + optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', @@ -86,13 +102,15 @@ module API optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :source_branch, type: String, desc: 'Return merge requests with the given source branch' optional :target_branch, type: String, desc: 'Return merge requests with the given target branch' optional :search, type: String, desc: 'Search merge requests for text present in the title or description' + optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' use :pagination end end @@ -117,7 +135,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of group merge requests' do success Entities::MergeRequestBasic end @@ -136,7 +154,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do include TimeTrackingEndpoints helpers do @@ -161,8 +179,9 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - optional :allow_maintainer_to_push, type: Boolean, desc: 'Whether a maintainer of the target project can push to the source project' + optional :remove_source_branch, type: Boolean, desc: 'Delete source branch when merging' + optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch' + optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration' optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' use :optional_params_ee @@ -231,6 +250,9 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' + optional :render_html, type: Boolean, desc: 'Returns the description and title rendered HTML' + optional :include_diverged_commits_count, type: Boolean, desc: 'Returns the commits count behind the target branch' + optional :include_rebase_in_progress, type: Boolean, desc: 'Returns whether a rebase operation is ongoing ' end desc 'Get a single merge request' do success Entities::MergeRequest @@ -238,7 +260,13 @@ module API get ':id/merge_requests/:merge_request_iid' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + present merge_request, + with: Entities::MergeRequest, + current_user: current_user, + project: user_project, + render_html: params[:render_html], + include_diverged_commits_count: params[:include_diverged_commits_count], + include_rebase_in_progress: params[:include_rebase_in_progress] end desc 'Get the participants of a merge request' do @@ -370,6 +398,19 @@ module API .cancel(merge_request) end + desc 'Rebase the merge request against its target branch' do + detail 'This feature was added in GitLab 11.6' + end + put ':id/merge_requests/:merge_request_iid/rebase' do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + authorize_push_to_merge_request!(merge_request) + + RebaseWorker.perform_async(merge_request.id, current_user.id) + + status :accepted + end + desc 'List issues that will be closed on merge' do success Entities::MRNote end @@ -378,7 +419,7 @@ module API end get ':id/merge_requests/:merge_request_iid/closes_issues' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user)) issues = paginate(issues) external_issues, internal_issues = issues.partition { |issue| issue.is_a?(ExternalIssue) } diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index a8eb137e46a..a0ca39b69d4 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module MilestoneResponses extend ActiveSupport::Concern diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 32b77aedba8..3cc09f6ac3f 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,23 +1,40 @@ +# frozen_string_literal: true + module API class Namespaces < Grape::API include PaginationParams before { authenticate! } + helpers do + params :optional_list_params_ee do + # EE::API::Namespaces would override this helper + end + + # EE::API::Namespaces would override this method + def custom_namespace_present_options + {} + end + end + resource :namespaces do desc 'Get a namespaces list' do success Entities::Namespace end params do optional :search, type: String, desc: "Search query for namespaces" + use :pagination + use :optional_list_params_ee end get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces namespaces = namespaces.search(params[:search]) if params[:search].present? - present paginate(namespaces), with: Entities::Namespace, current_user: current_user + options = { with: Entities::Namespace, current_user: current_user } + + present paginate(namespaces), options.reverse_merge(custom_namespace_present_options) end desc 'Get a namespace by ID' do @@ -26,7 +43,7 @@ module API params do requires :id, type: String, desc: "Namespace's ID or path" end - get ':id' do + get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do present user_namespace, with: Entities::Namespace, current_user: current_user end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 39923e6d5b5..1bdf7aeb119 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Notes < Grape::API include PaginationParams @@ -14,7 +16,7 @@ module API params do requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do noteables_str = noteable_type.to_s.underscore.pluralize desc "Get a list of #{noteable_type.to_s.downcase} notes" do @@ -28,6 +30,7 @@ module API desc: 'Return notes sorted in `asc` or `desc` order.' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/#{noteables_str}/:noteable_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) @@ -45,6 +48,7 @@ module API .reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note end + # rubocop: enable CodeReuse/ActiveRecord desc "Get a single #{noteable_type.to_s.downcase} note" do success Entities::Note diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 0266bf2f717..8cb46bd3ad6 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # notification_settings API class NotificationSettings < Grape::API @@ -23,7 +25,7 @@ module API params do optional :level, type: String, desc: 'The global notification level' optional :notification_email, type: String, desc: 'The email address to send notifications' - NotificationSetting::EMAIL_EVENTS.each do |event| + NotificationSetting.email_events.each do |event| optional event, type: Boolean, desc: 'Enable/disable this notification' end end @@ -50,11 +52,13 @@ module API end end - %w[group project].each do |source_type| + [Group, Project].each do |source_class| + source_type = source_class.name.underscore + params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting @@ -73,7 +77,7 @@ module API end params do optional :level, type: String, desc: "The #{source_type} notification level" - NotificationSetting::EMAIL_EVENTS.each do |event| + NotificationSetting.email_events(source_class).each do |event| optional event, type: Boolean, desc: 'Enable/disable this notification' end end diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index ba33993d852..78442f465bd 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module API class PagesDomains < Grape::API include PaginationParams - PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) + PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) before do authenticate! @@ -13,9 +15,11 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_pages_domain! user_project.pages_domains.find_by(domain: params[:domain]) || not_found!('PagesDomain') end + # rubocop: enable CodeReuse/ActiveRecord def pages_domain @pages_domain ||= find_pages_domain! @@ -50,7 +54,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do require_pages_enabled! end @@ -61,11 +65,13 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ":id/pages/domains" do authorize! :read_pages, user_project present paginate(user_project.pages_domains.order(:domain)), with: Entities::PagesDomain end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single pages domain' do success Entities::PagesDomain diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb index f566eb3ed2b..ae03595eb25 100644 --- a/lib/api/pagination_params.rb +++ b/lib/api/pagination_params.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Concern for declare pagination params. # diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 37f32411296..c86b50d3736 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class PipelineSchedules < Grape::API include PaginationParams @@ -7,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all pipeline schedules' do success Entities::PipelineSchedule end @@ -16,6 +18,7 @@ module API optional :scope, type: String, values: %w[active inactive], desc: 'The scope of pipeline schedules' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/pipeline_schedules' do authorize! :read_pipeline_schedule, user_project @@ -23,12 +26,13 @@ module API .preload([:owner, :last_pipeline]) present paginate(schedules), with: Entities::PipelineSchedule end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single pipeline schedule' do success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end get ':id/pipeline_schedules/:pipeline_schedule_id' do present pipeline_schedule, with: Entities::PipelineScheduleDetails @@ -39,7 +43,7 @@ module API end params do requires :description, type: String, desc: 'The description of pipeline schedule' - requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false requires :cron, type: String, desc: 'The cron' optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' @@ -83,7 +87,7 @@ module API success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -99,7 +103,7 @@ module API success Entities::PipelineScheduleDetails end params do - requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' end delete ':id/pipeline_schedules/:pipeline_schedule_id' do authorize! :admin_pipeline_schedule, pipeline_schedule @@ -161,6 +165,7 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def pipeline_schedule @pipeline_schedule ||= user_project @@ -172,7 +177,9 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def pipeline_schedule_variable @pipeline_schedule_variable ||= pipeline_schedule.variables.find_by(key: params[:key]).tap do |pipeline_schedule_variable| @@ -181,6 +188,7 @@ module API end end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 735591fedd5..ac8fe98e55e 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Pipelines < Grape::API include PaginationParams @@ -7,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::PipelineBasic @@ -31,7 +33,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project, params).execute + pipelines = PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic end @@ -40,16 +42,22 @@ module API success Entities::Pipeline end params do - requires :ref, type: String, desc: 'Reference' + requires :ref, type: String, desc: 'Reference' + optional :variables, Array, desc: 'Array of variables available in the pipeline' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/pipeline' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') authorize! :create_pipeline, user_project + pipeline_params = declared_params(include_missing: false) + .merge(variables_attributes: params[:variables]) + .except(:variables) + new_pipeline = Ci::CreatePipelineService.new(user_project, current_user, - declared_params(include_missing: false)) + pipeline_params) .execute(:api, ignore_skip_ci: true, save_on_errors: false) if new_pipeline.persisted? @@ -58,6 +66,7 @@ module API render_validation_error!(new_pipeline) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Gets a specific pipeline for the project' do detail 'This feature was introduced in GitLab 8.11' @@ -67,20 +76,35 @@ module API requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end get ':id/pipelines/:pipeline_id' do - authorize! :read_pipeline, user_project + authorize! :read_pipeline, pipeline present pipeline, with: Entities::Pipeline end + desc 'Deletes a pipeline' do + detail 'This feature was introduced in GitLab 11.6' + http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']] + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + delete ':id/pipelines/:pipeline_id' do + authorize! :destroy_pipeline, pipeline + + destroy_conditionally!(pipeline) do + ::Ci::DestroyPipelineService.new(user_project, current_user).execute(pipeline) + end + end + desc 'Retry builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Pipeline end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end post ':id/pipelines/:pipeline_id/retry' do - authorize! :update_pipeline, user_project + authorize! :update_pipeline, pipeline pipeline.retry_failed(current_user) @@ -92,10 +116,10 @@ module API success Entities::Pipeline end params do - requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end post ':id/pipelines/:pipeline_id/cancel' do - authorize! :update_pipeline, user_project + authorize! :update_pipeline, pipeline pipeline.cancel_running @@ -106,7 +130,7 @@ module API helpers do def pipeline - @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + @pipeline ||= user_project.ci_pipelines.find(params[:pipeline_id]) end end end diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb new file mode 100644 index 00000000000..c96261a7b57 --- /dev/null +++ b/lib/api/project_clusters.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true + +module API + class ProjectClusters < Grape::API + include PaginationParams + + before { authenticate! } + + # EE::API::ProjectClusters will + # override these methods + helpers do + params :create_params_ee do + end + + params :update_params_ee do + end + end + + params do + requires :id, type: String, desc: 'The ID of the project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all clusters from the project' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Cluster + end + params do + use :pagination + end + get ':id/clusters' do + authorize! :read_cluster, user_project + + present paginate(clusters_for_current_user), with: Entities::Cluster + end + + desc 'Get specific cluster for the project' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + end + get ':id/clusters/:cluster_id' do + authorize! :read_cluster, cluster + + present cluster, with: Entities::ClusterProject + end + + desc 'Adds an existing cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' + requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' + requires :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Project' + optional :authorization_type, type: String, values: Clusters::Platforms::Kubernetes.authorization_types.keys, default: 'rbac', desc: 'Cluster authorization type, defaults to RBAC' + end + use :create_params_ee + end + post ':id/clusters/user' do + authorize! :add_cluster, user_project, 'Instance does not support multiple Kubernetes clusters' + + user_cluster = ::Clusters::CreateService + .new(current_user, create_cluster_user_params) + .execute + + if user_cluster.persisted? + present user_cluster, with: Entities::ClusterProject + else + render_validation_error!(user_cluster) + end + end + + desc 'Update an existing cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The cluster ID' + optional :name, type: String, desc: 'Cluster name' + optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do + optional :api_url, type: String, desc: 'URL to access the Kubernetes API' + optional :token, type: String, desc: 'Token to authenticate against Kubernetes' + optional :ca_cert, type: String, desc: 'TLS certificate (needed if API is using a self-signed TLS certificate)' + optional :namespace, type: String, desc: 'Unique namespace related to Project' + end + use :update_params_ee + end + put ':id/clusters/:cluster_id' do + authorize! :update_cluster, cluster + + update_service = Clusters::UpdateService.new(current_user, update_cluster_params) + + if update_service.execute(cluster) + present cluster, with: Entities::ClusterProject + else + render_validation_error!(cluster) + end + end + + desc 'Remove a cluster' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::ClusterProject + end + params do + requires :cluster_id, type: Integer, desc: 'The Cluster ID' + end + delete ':id/clusters/:cluster_id' do + authorize! :admin_cluster, cluster + + destroy_conditionally!(cluster) + end + end + + helpers do + def clusters_for_current_user + @clusters_for_current_user ||= ClustersFinder.new(user_project, current_user, :all).execute + end + + def cluster + @cluster ||= clusters_for_current_user.find(params[:cluster_id]) + end + + def create_cluster_user_params + declared_params.merge({ + provider_type: :user, + platform_type: :kubernetes, + clusterable: user_project + }) + end + + def update_cluster_params + declared_params(include_missing: false).without(:cluster_id) + end + end + end +end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 5ef4e9d530c..e34ed0bdb44 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectExport < Grape::API before do @@ -21,11 +23,11 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - render_api_error!('404 Not found or has expired', 404) unless path - - present_disk_file!(path, File.basename(path), 'application/gzip') + if user_project.export_file_exists? + present_carrierwave_file!(user_project.export_file) + else + render_api_error!('404 Not found or has expired', 404) + end end desc 'Start export' do diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 68921ae439b..0e7576c9243 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectHooks < Grape::API include PaginationParams @@ -20,13 +22,14 @@ module API optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project hooks' do success Entities::ProjectHook end @@ -63,6 +66,7 @@ module API present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? + error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? not_found!("Project hook #{hook.errors.messages}") end @@ -80,10 +84,11 @@ module API update_params = declared_params(include_missing: false) - if hook.update_attributes(update_params) + if hook.update(update_params) present hook, with: Entities::ProjectHook else error!("Invalid url given", 422) if hook.errors[:url].present? + error!("Invalid branch filter given", 422) if hook.errors[:push_events_branch_filter].present? not_found!("Project hook #{hook.errors.messages}") end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index bc5152e539f..c64ec2fcc95 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectImport < Grape::API include PaginationParams @@ -21,7 +23,7 @@ module API forbidden! unless Gitlab::CurrentSettings.import_sources.include?('gitlab_project') end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do requires :path, type: String, desc: 'The new project path and name' requires :file, type: File, desc: 'The project export file to be imported' diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 306dc0e63d7..da31bcb8dac 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectMilestones < Grape::API include PaginationParams @@ -10,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of project milestones' do success Entities::Milestone end @@ -64,7 +66,8 @@ module API delete ":id/milestones/:milestone_id" do authorize! :admin_milestone, user_project - user_project.milestones.find(params[:milestone_id]).destroy + milestone = user_project.milestones.find(params[:milestone_id]) + Milestones::DestroyService.new(user_project, current_user).execute(milestone) status(204) end diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index 71005acc587..175fbb2ce92 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnapshots < Grape::API helpers ::API::Helpers::ProjectSnapshotsHelpers diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 1de5551fee9..a607df411a6 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class ProjectSnippets < Grape::API include PaginationParams @@ -7,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -49,7 +51,7 @@ module API params do requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' - requires :code, type: String, desc: 'The content of the snippet' + requires :code, type: String, allow_blank: false, desc: 'The content of the snippet' optional :description, type: String, desc: 'The description of a snippet' requires :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, @@ -78,13 +80,14 @@ module API requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' - optional :code, type: String, desc: 'The content of the snippet' + optional :code, type: String, allow_blank: false, desc: 'The content of the snippet' optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :code, :visibility_level end + # rubocop: disable CodeReuse/ActiveRecord put ":id/snippets/:snippet_id" do snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) not_found!('Snippet') unless snippet @@ -107,11 +110,13 @@ module API render_validation_error!(snippet) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a project snippet' params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/snippets/:snippet_id" do snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) not_found!('Snippet') unless snippet @@ -120,11 +125,13 @@ module API destroy_conditionally!(snippet) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a raw project snippet' params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/snippets/:snippet_id/raw" do snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) not_found!('Snippet') unless snippet @@ -133,6 +140,7 @@ module API content_type 'text/plain' present snippet.content end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the user agent details for a project snippet' do success Entities::UserAgentDetail @@ -140,6 +148,7 @@ module API params do requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/snippets/:snippet_id/user_agent_detail" do authenticated_as_admin! @@ -149,6 +158,7 @@ module API present snippet.user_agent_detail, with: Entities::UserAgentDetail end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb new file mode 100644 index 00000000000..d05ddad7466 --- /dev/null +++ b/lib/api/project_templates.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module API + class ProjectTemplates < Grape::API + include PaginationParams + + TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses].freeze + + before { authenticate_non_get! } + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses) of the template' + end + resource :projects do + desc 'Get a list of templates available to this project' do + detail 'This endpoint was introduced in GitLab 11.4' + end + params do + use :pagination + end + get ':id/templates/:type' do + templates = TemplateFinder + .build(params[:type], user_project) + .execute + + present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList + end + + desc 'Download a template available to this project' do + detail 'This endpoint was introduced in GitLab 11.4' + end + params do + requires :name, type: String, desc: 'The name of the template' + + optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' + optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' + end + get ':id/templates/:type/:name', requirements: { name: /[\w\.-]+/ } do + template = TemplateFinder + .build(params[:type], user_project, name: params[:name]) + .execute + + not_found!('Template') unless template.present? + + template.resolve!( + project_name: params[:project].presence, + fullname: params[:fullname].presence || current_user&.name + ) + + if template.is_a?(::LicenseTemplate) + present template, with: Entities::License + else + present template, with: Entities::Template + end + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ef3680c5d9..3afa2d8a6b0 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'declarative_policy' module API @@ -9,6 +11,56 @@ module API before { authenticate_non_get! } helpers do + params :optional_filter_params_ee do + # EE::API::Projects would override this helper + end + + params :optional_update_params_ee do + # EE::API::Projects would override this helper + end + + # EE::API::Projects would override this method + def apply_filters(projects) + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + projects = projects.with_statistics if params[:statistics] + + projects + end + + def verify_update_project_attrs!(project, attrs) + end + end + + def self.update_params_at_least_one_of + [ + :jobs_enabled, + :resolve_outdated_diff_discussions, + :ci_config_path, + :container_registry_enabled, + :default_branch, + :description, + :issues_enabled, + :lfs_enabled, + :merge_requests_enabled, + :merge_method, + :name, + :only_allow_merge_if_all_discussions_are_resolved, + :only_allow_merge_if_pipeline_succeeds, + :path, + :printing_merge_request_link_enabled, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, + :tag_list, + :visibility, + :wiki_enabled, + :avatar + ] + end + + helpers do params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end @@ -30,7 +82,7 @@ module API end params :filter_params do - optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :archived, type: Boolean, desc: 'Limit by archived status' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of projects matching the search criteria' @@ -39,6 +91,9 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' + optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user' + + use :optional_filter_params_ee end params :create_params do @@ -52,16 +107,15 @@ module API def present_projects(projects, options = {}) projects = reorder_projects(projects) - projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] - projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] - projects = projects.with_statistics if params[:statistics] + projects = apply_filters(projects) projects = paginate(projects) projects, options = with_custom_attributes(projects, options) options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - current_user: current_user + current_user: current_user, + license: false ) options[:with] = Entities::BasicProjectDetails if params[:simple] @@ -74,7 +128,7 @@ module API end end - resource :users, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :users, requirements: API::USER_REQUIREMENTS do desc 'Get a user projects' do success Entities::BasicProjectDetails end @@ -147,6 +201,7 @@ module API use :optional_project_params use :create_params end + # rubocop: disable CodeReuse/ActiveRecord post "user/:user_id" do authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) @@ -163,25 +218,30 @@ module API render_validation_error!(project) end end + # rubocop: enable CodeReuse/ActiveRecord end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a single project' do success Entities::ProjectWithAccess end params do use :statistics_params use :with_custom_attributes + + optional :license, type: Boolean, default: false, + desc: 'Include project license data' end get ":id" do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, user_can_admin_project: can?(current_user, :admin_project, user_project), - statistics: params[:statistics] + statistics: params[:statistics], + license: params[:license] } project, options = with_custom_attributes(user_project, options) @@ -232,42 +292,23 @@ module API present_projects forks end + desc 'Check pages access of this project' + get ':id/pages_access' do + authorize! :read_pages_content, user_project unless user_project.public_pages? + status 200 + end + desc 'Update an existing project' do success Entities::Project end params do - # CE - at_least_one_of_ce = - [ - :jobs_enabled, - :resolve_outdated_diff_discussions, - :ci_config_path, - :container_registry_enabled, - :default_branch, - :description, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :merge_method, - :name, - :only_allow_merge_if_all_discussions_are_resolved, - :only_allow_merge_if_pipeline_succeeds, - :path, - :printing_merge_request_link_enabled, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :tag_list, - :visibility, - :wiki_enabled - ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' optional :path, type: String, desc: 'The path of the repository' use :optional_project_params - at_least_one_of(*at_least_one_of_ce) + + at_least_one_of(*::API::Projects.update_params_at_least_one_of) end put ':id' do authorize_admin_project @@ -277,6 +318,8 @@ module API attrs = translate_params_for_compatibility(attrs) + verify_update_project_attrs!(user_project, attrs) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success @@ -293,7 +336,7 @@ module API post ':id/archive' do authorize!(:archive_project, user_project) - user_project.archive! + ::Projects::UpdateService.new(user_project, current_user, archived: true).execute present user_project, with: Entities::Project end @@ -304,7 +347,7 @@ module API post ':id/unarchive' do authorize!(:archive_project, user_project) - user_project.unarchive! + ::Projects::UpdateService.new(@project, current_user, archived: false).execute present user_project, with: Entities::Project end @@ -358,7 +401,7 @@ module API requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' end post ":id/fork/:forked_from_id" do - authenticated_as_admin! + authorize! :admin_project, user_project fork_from_project = find_project!(params[:forked_from_id]) @@ -416,6 +459,7 @@ module API params do requires :group_id, type: Integer, desc: 'The ID of the group' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id/share/:group_id" do authorize! :admin_project, user_project @@ -424,13 +468,14 @@ module API destroy_conditionally!(link) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Upload a file' params do requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute.to_h end desc 'Get the users list of a project' do @@ -459,6 +504,23 @@ module API conflict!(error.message) end end + + desc 'Transfer a project to a new namespace' + params do + requires :namespace, type: String, desc: 'The ID or path of the new namespace' + end + put ":id/transfer" do + authorize! :change_namespace, user_project + + namespace = find_namespace!(params[:namespace]) + result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) + + if result + present user_project, with: Entities::Project + else + render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) + end + end end end end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 6482fd94ab8..263468c9aa6 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -1,15 +1,17 @@ +# frozen_string_literal: true + module API module ProjectsRelationBuilder extend ActiveSupport::Concern - module ClassMethods + class_methods do def prepare_relation(projects_relation, options = {}) projects_relation = preload_relation(projects_relation, options) execute_batch_counting(projects_relation) projects_relation end - def preload_relation(projects_relation, options = {}) + def preload_relation(projects_relation, options = {}) projects_relation end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index aa7cab4a741..5af43448727 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -1,26 +1,30 @@ +# frozen_string_literal: true + module API class ProtectedBranches < Grape::API include PaginationParams - BRANCH_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) + BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) before { authorize_admin_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Get a project's protected branches" do success Entities::ProtectedBranch end params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches' do protected_branches = user_project.protected_branches.preload(:push_access_levels, :merge_access_levels) present paginate(protected_branches), with: Entities::ProtectedBranch, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single protected branch' do success Entities::ProtectedBranch @@ -28,11 +32,13 @@ module API params do requires :name, type: String, desc: 'The name of the branch or wildcard' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) present protected_branch, with: Entities::ProtectedBranch, project: user_project end + # rubocop: enable CodeReuse/ActiveRecord desc 'Protect a single branch or wildcard' do success Entities::ProtectedBranch @@ -40,12 +46,13 @@ module API params do requires :name, type: String, desc: 'The name of the protected branch' optional :push_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, - desc: 'Access levels allowed to push (defaults: `40`, master access level)' + values: ProtectedBranch::PushAccessLevel.allowed_access_levels, + desc: 'Access levels allowed to push (defaults: `40`, maintainer access level)' optional :merge_access_level, type: Integer, - values: ProtectedRefAccess::ALLOWED_ACCESS_LEVELS, - desc: 'Access levels allowed to merge (defaults: `40`, master access level)' + values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, + desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/protected_branches' do protected_branch = user_project.protected_branches.find_by(name: params[:name]) if protected_branch @@ -62,11 +69,13 @@ module API render_api_error!(protected_branch.errors.full_messages, 422) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Unprotect a single branch' params do requires :name, type: String, desc: 'The name of the protected branch' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/protected_branches/:name', requirements: BRANCH_ENDPOINT_REQUIREMENTS do protected_branch = user_project.protected_branches.find_by!(name: params[:name]) @@ -75,6 +84,7 @@ module API destroy_service.execute(protected_branch) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb new file mode 100644 index 00000000000..ee13473c848 --- /dev/null +++ b/lib/api/protected_tags.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module API + class ProtectedTags < Grape::API + include PaginationParams + + TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_admin_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "Get a project's protected tags" do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + use :pagination + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/protected_tags' do + protected_tags = user_project.protected_tags.preload(:create_access_levels) + + present paginate(protected_tags), with: Entities::ProtectedTag, project: user_project + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Get a single protected tag' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + requires :name, type: String, desc: 'The name of the tag or wildcard' + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do + protected_tag = user_project.protected_tags.find_by!(name: params[:name]) + + present protected_tag, with: Entities::ProtectedTag, project: user_project + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Protect a single tag or wildcard' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::ProtectedTag + end + params do + requires :name, type: String, desc: 'The name of the protected tag' + optional :create_access_level, type: Integer, default: Gitlab::Access::MAINTAINER, + values: ProtectedTag::CreateAccessLevel.allowed_access_levels, + desc: 'Access levels allowed to create (defaults: `40`, maintainer access level)' + end + post ':id/protected_tags' do + protected_tags_params = { + name: params[:name], + create_access_levels_attributes: [{ access_level: params[:create_access_level] }] + } + + protected_tag = ::ProtectedTags::CreateService.new(user_project, + current_user, + protected_tags_params).execute + + if protected_tag.persisted? + present protected_tag, with: Entities::ProtectedTag, project: user_project + else + render_api_error!(protected_tag.errors.full_messages, 422) + end + end + + desc 'Unprotect a single tag' do + detail 'This feature was introduced in GitLab 11.3.' + end + params do + requires :name, type: String, desc: 'The name of the protected tag' + end + # rubocop: disable CodeReuse/ActiveRecord + delete ':id/protected_tags/:name', requirements: TAG_ENDPOINT_REQUIREMENTS do + protected_tag = user_project.protected_tags.find_by!(name: params[:name]) + + destroy_conditionally!(protected_tag) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb new file mode 100644 index 00000000000..e3072684ef7 --- /dev/null +++ b/lib/api/release/links.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module API + module Release + class Links < Grape::API + include PaginationParams + + RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + resource 'releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + resource :assets do + desc 'Get a list of links of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + use :pagination + end + get 'links' do + authorize! :read_release, release + + present paginate(release.links.sorted), with: Entities::Releases::Link + end + + desc 'Create a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + requires :name, type: String, desc: 'The name of the link' + requires :url, type: String, desc: 'The URL of the link' + end + post 'links' do + authorize! :create_release, release + + new_link = release.links.create(declared_params(include_missing: false)) + + if new_link.persisted? + present new_link, with: Entities::Releases::Link + else + render_api_error!(new_link.errors.messages, 400) + end + end + + params do + requires :link_id, type: String, desc: 'The id of the link' + end + resource 'links/:link_id' do + desc 'Get a link detail of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + get do + authorize! :read_release, release + + present link, with: Entities::Releases::Link + end + + desc 'Update a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + params do + optional :name, type: String, desc: 'The name of the link' + optional :url, type: String, desc: 'The URL of the link' + at_least_one_of :name, :url + end + put do + authorize! :update_release, release + + if link.update(declared_params(include_missing: false)) + present link, with: Entities::Releases::Link + else + render_api_error!(link.errors.messages, 400) + end + end + + desc 'Delete a link of a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Releases::Link + end + delete do + authorize! :destroy_release, release + + if link.destroy + present link, with: Entities::Releases::Link + else + render_api_error!(link.errors.messages, 400) + end + end + end + end + end + end + + helpers do + def release + @release ||= user_project.releases.find_by_tag!(params[:tag]) + end + + def link + @link ||= release.links.find(params[:link_id]) + end + end + end + end +end diff --git a/lib/api/releases.rb b/lib/api/releases.rb new file mode 100644 index 00000000000..cb85028f22c --- /dev/null +++ b/lib/api/releases.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module API + class Releases < Grape::API + include PaginationParams + + RELEASE_ENDPOINT_REQUIREMETS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + + before { authorize_read_releases! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get a project releases' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + use :pagination + end + get ':id/releases' do + releases = ::ReleasesFinder.new(user_project, current_user).execute + + present paginate(releases), with: Entities::Release + end + + desc 'Get a single project release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_read_release! + + present release, with: Entities::Release + end + + desc 'Create a new release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + requires :name, type: String, desc: 'The name of the release' + requires :description, type: String, desc: 'The release notes' + optional :ref, type: String, desc: 'The commit sha or branch name' + optional :assets, type: Hash do + optional :links, type: Array do + requires :name, type: String + requires :url, type: String + end + end + end + post ':id/releases' do + authorize_create_release! + + result = ::Releases::CreateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Update a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + optional :name, type: String, desc: 'The name of the release' + optional :description, type: String, desc: 'Release notes with markdown support' + end + put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_update_release! + + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Delete a release' do + detail 'This feature was introduced in GitLab 11.7.' + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + end + delete ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do + authorize_destroy_release! + + result = ::Releases::DestroyService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if result[:status] == :success + present result[:release], with: Entities::Release + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_read_releases! + authorize! :read_release, user_project + end + + def authorize_read_release! + authorize! :read_release, release + end + + def authorize_update_release! + authorize! :update_release, release + end + + def authorize_destroy_release! + authorize! :destroy_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end + end +end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index bb3fa99af38..32e05d84491 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mime/types' module API @@ -9,7 +11,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -100,9 +102,10 @@ module API params do requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to], straight: params[:straight]) present compare, with: Entities::Compare end @@ -122,6 +125,34 @@ module API not_found! end end + + desc 'Get the common ancestor between commits' do + success Entities::Commit + end + params do + requires :refs, type: Array[String] + end + get ':id/repository/merge_base' do + refs = params[:refs] + + if refs.size < 2 + render_api_error!('Provide at least 2 refs', 400) + end + + merge_base = Gitlab::Git::MergeBase.new(user_project.repository, refs) + + if merge_base.unknown_refs.any? + ref_noun = 'ref'.pluralize(merge_base.unknown_refs.size) + message = "Could not find #{ref_noun}: #{merge_base.unknown_refs.join(', ')}" + render_api_error!(message, 400) + end + + if merge_base.commit + present merge_base.commit, with: Entities::Commit + else + not_found!("Merge Base") + end + end end end end diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb new file mode 100644 index 00000000000..0c328f7268e --- /dev/null +++ b/lib/api/resource_label_events.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module API + class ResourceLabelEvents < Grape::API + include PaginationParams + helpers ::API::Helpers::NotesHelpers + + before { authenticate! } + + EVENTABLE_TYPES = [Issue, MergeRequest].freeze + + EVENTABLE_TYPES.each do |eventable_type| + parent_type = eventable_type.parent_class.to_s.underscore + eventables_str = eventable_type.to_s.underscore.pluralize + + params do + requires :id, type: String, desc: "The ID of a #{parent_type}" + end + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + use :pagination + end + + # rubocop: disable CodeReuse/ActiveRecord + get ":id/#{eventables_str}/:eventable_id/resource_label_events" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + events = eventable.resource_label_events.includes(:label, :user) + + present paginate(events), with: Entities::ResourceLabelEvent + end + # rubocop: enable CodeReuse/ActiveRecord + + desc "Get a single #{eventable_type.to_s.downcase} resource label event" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :event_id, type: String, desc: 'The ID of a resource label event' + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + end + get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + event = eventable.resource_label_events.find(params[:event_id]) + + present event, with: Entities::ResourceLabelEvent + end + end + end + end +end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index e9886c76870..c60d25b88cb 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runner < Grape::API helpers ::API::Helpers::Runner @@ -24,13 +26,13 @@ module API attributes = if runner_registration_token_valid? # Create shared runner. Requires admin access - attributes.merge(is_shared: true, runner_type: :instance_type) - elsif project = Project.find_by(runners_token: params[:token]) + attributes.merge(runner_type: :instance_type) + elsif project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project - attributes.merge(is_shared: false, runner_type: :project_type, projects: [project]) - elsif group = Group.find_by(runners_token: params[:token]) + attributes.merge(runner_type: :project_type, projects: [project]) + elsif group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group - attributes.merge(is_shared: false, runner_type: :group_type, groups: [group]) + attributes.merge(runner_type: :group_type, groups: [group]) else forbidden! end @@ -80,26 +82,44 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) optional :last_update, type: String, desc: %q(Runner's queue last_update token) - optional :info, type: Hash, desc: %q(Runner's metadata) + optional :info, type: Hash, desc: %q(Runner's metadata) do + optional :name, type: String, desc: %q(Runner's name) + optional :version, type: String, desc: %q(Runner's version) + optional :revision, type: String, desc: %q(Runner's revision) + optional :platform, type: String, desc: %q(Runner's platform) + optional :architecture, type: String, desc: %q(Runner's architecture) + optional :executor, type: String, desc: %q(Runner's executor) + optional :features, type: Hash, desc: %q(Runner's features) + end + optional :session, type: Hash, desc: %q(Runner's session data) do + optional :url, type: String, desc: %q(Session's url) + optional :certificate, type: String, desc: %q(Session's certificate) + optional :authorization, type: String, desc: %q(Session's authorization) + end end post '/request' do authenticate_runner! - no_content! unless current_runner.active? - if current_runner.runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] + unless current_runner.active? + header 'X-GitLab-Last-Update', current_runner.ensure_runner_queue_value + break no_content! + end + + runner_params = declared_params(include_missing: false) + + if current_runner.runner_queue_value_latest?(runner_params[:last_update]) + header 'X-GitLab-Last-Update', runner_params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) break no_content! end new_update = current_runner.ensure_runner_queue_value - result = ::Ci::RegisterJobService.new(current_runner).execute + result = ::Ci::RegisterJobService.new(current_runner).execute(runner_params) if result.valid? if result.build - Gitlab::Metrics.add_event(:build_found, - project: result.build.project.full_path) - present result.build, with: Entities::JobRequest::Response + Gitlab::Metrics.add_event(:build_found) + present Ci::BuildRunnerPresenter.new(result.build), with: Entities::JobRequest::Response else Gitlab::Metrics.add_event(:build_not_found) header 'X-GitLab-Last-Update', new_update @@ -120,19 +140,19 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) - optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, - desc: %q(Job's failure_reason) + optional :failure_reason, type: String, desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! - forbidden!('Job is not running') unless job.running? + job_forbidden!(job, 'Job is not running') unless job.running? job.trace.set(params[:trace]) if params[:trace] - Gitlab::Metrics.add_event(:update_build, - project: job.project.full_path) + Gitlab::Metrics.add_event(:update_build) case params[:state].to_s + when 'running' + job.touch if job.needs_touch? when 'success' job.success! when 'failed' @@ -152,7 +172,7 @@ module API end patch '/:id/trace' do job = authenticate_job! - forbidden!('Job is not running') unless job.running? + job_forbidden!(job, 'Job is not running') unless job.running? error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] @@ -205,7 +225,7 @@ module API status 200 content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - JobArtifactUploader.workhorse_authorize + JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size) end desc 'Upload artifacts for job' do @@ -220,6 +240,10 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) + optional :artifact_type, type: String, desc: %q(The type of artifact), + default: 'archive', values: Ci::JobArtifact.file_types.keys + optional :artifact_format, type: String, desc: %q(The format of artifact), + default: 'zip', values: Ci::JobArtifact.file_formats.keys optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) @@ -243,29 +267,29 @@ module API bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size - bad_request!("Already uploaded") if job.job_artifacts_archive - expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.build_job_artifacts_archive( + job.job_artifacts.build( project: job.project, file: artifacts, - file_type: :archive, + file_type: params['artifact_type'], + file_format: params['artifact_format'], file_sha256: artifacts.sha256, expire_in: expire_in) if metadata - job.build_job_artifacts_metadata( + job.job_artifacts.build( project: job.project, file: metadata, file_type: :metadata, + file_format: :gzip, file_sha256: metadata.sha256, expire_in: expire_in) end if job.update(artifacts_expire_in: expire_in) - present job, with: Entities::JobRequest::Response + present Ci::BuildRunnerPresenter.new(job), with: Entities::JobRequest::Response else render_validation_error!(job) end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 2b78075ddbf..f72b33605a7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Runners < Grape::API include PaginationParams @@ -9,12 +11,20 @@ module API success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_STATUSES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get do - runners = filter_runners(current_user.ci_owned_runners, params[:scope], without: %w(specific shared)) + runners = current_user.ci_owned_runners + runners = filter_runners(runners, params[:scope], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -22,13 +32,22 @@ module API success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online specific shared], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get 'all' do authenticated_as_admin! - runners = filter_runners(Ci::Runner.all, params[:scope]) + + runners = Ci::Runner.all + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -58,7 +77,7 @@ module API optional :access_level, type: String, values: Ci::Runner.access_levels.keys, desc: 'The access_level of the runner' optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' - at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked, :access_level, :maximum_timeout end put ':id' do runner = get_runner(params.delete(:id)) @@ -94,7 +113,7 @@ module API optional :status, type: String, desc: 'Status of the job', values: Ci::Build::AVAILABLE_STATUSES use :pagination end - get ':id/jobs' do + get ':id/jobs' do runner = get_runner(params[:id]) authenticate_list_runners_jobs!(runner) @@ -107,19 +126,27 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authorize_admin_project } desc 'Get runners available for project' do success Entities::Runner end params do - optional :scope, type: String, values: %w[active paused online specific shared], + optional :scope, type: String, values: Ci::Runner::AVAILABLE_SCOPES, desc: 'The scope of specific runners to show' + optional :type, type: String, values: Ci::Runner::AVAILABLE_TYPES, + desc: 'The type of the runners to show' + optional :status, type: String, values: Ci::Runner::AVAILABLE_STATUSES, + desc: 'The status of the runners to show' use :pagination end get ':id/runners' do - runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) + runners = Ci::Runner.owned_or_instance_wide(user_project.id) + runners = filter_runners(runners, params[:scope]) + runners = filter_runners(runners, params[:type], allowed_scopes: Ci::Runner::AVAILABLE_TYPES) + runners = filter_runners(runners, params[:status], allowed_scopes: Ci::Runner::AVAILABLE_STATUSES) + present paginate(runners), with: Entities::Runner end @@ -146,6 +173,7 @@ module API params do requires :runner_id, type: Integer, desc: 'The ID of the runner' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/runners/:runner_id' do runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) not_found!('Runner') unless runner_project @@ -155,19 +183,20 @@ module API destroy_conditionally!(runner_project) end + # rubocop: enable CodeReuse/ActiveRecord end helpers do - def filter_runners(runners, scope, options = {}) + def filter_runners(runners, scope, allowed_scopes: ::Ci::Runner::AVAILABLE_SCOPES) return runners unless scope.present? - available_scopes = ::Ci::Runner::AVAILABLE_SCOPES - if options[:without] - available_scopes = available_scopes - options[:without] + unless allowed_scopes.include?(scope) + render_api_error!('Scope contains invalid value', 400) end - if (available_scopes & [scope]).empty? - render_api_error!('Scope contains invalid value', 400) + # Support deprecated scopes + if runners.respond_to?("deprecated_#{scope}") + scope = "deprecated_#{scope}" end runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend @@ -180,7 +209,7 @@ module API end def authenticate_show_runner!(runner) - return if runner.is_shared || current_user.admin? + return if runner.instance_type? || current_user.admin? forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end diff --git a/lib/api/scope.rb b/lib/api/scope.rb index d5165b2e482..707775e5d15 100644 --- a/lib/api/scope.rb +++ b/lib/api/scope.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Encapsulate a scope used for authorization, such as `api`, or `read_user` module API class Scope diff --git a/lib/api/search.rb b/lib/api/search.rb index 5d9ec617cb7..f5db692afe5 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Search < Grape::API include PaginationParams @@ -33,14 +35,7 @@ module API end def process_results(results) - case params[:scope] - when 'wiki_blobs' - paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob, user_project) } - when 'blobs' - paginate(results).map { |blob| blob[1] } - else - paginate(results) - end + paginate(results) end def snippets? @@ -70,7 +65,7 @@ module API end end - resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Search on GitLab' do detail 'This feature was introduced in GitLab 10.5.' end @@ -89,7 +84,7 @@ module API end end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Search on GitLab' do detail 'This feature was introduced in GitLab 10.5.' end diff --git a/lib/api/services.rb b/lib/api/services.rb index 794fdab8f2b..637b5a8a89a 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -298,6 +298,14 @@ module API desc: 'Title' } ], + 'discord' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'Discord webhook. e.g. https://discordapp.com/api/webhooks/…' + } + ], 'drone-ci' => [ { required: true, @@ -354,18 +362,12 @@ module API desc: 'Flowdock token' } ], - 'gemnasium' => [ + 'hangouts-chat' => [ { required: true, - name: :api_key, - type: String, - desc: 'Your personal API key on gemnasium.com' - }, - { - required: true, - name: :token, + name: :webhook, type: String, - desc: "The project's slug on gemnasium.com" + desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' } ], 'hipchat' => [ @@ -683,11 +685,12 @@ module API BuildkiteService, CampfireService, CustomIssueTrackerService, + DiscordService, DroneCiService, EmailsOnPushService, ExternalWikiService, FlowdockService, - GemnasiumService, + HangoutsChatService, HipchatService, IrkerService, JiraService, @@ -760,7 +763,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before { authenticate! } before { authorize_admin_project } @@ -787,7 +790,7 @@ module API service = user_project.find_or_initialize_service(service_slug.underscore) service_params = declared_params(include_missing: false).merge(active: true) - if service.update_attributes(service_params) + if service.update(service_params) present service, with: Entities::ProjectService else render_api_error!('400 Bad Request', 400) @@ -807,7 +810,7 @@ module API hash.merge!(key => nil) end - unless service.update_attributes(attrs.merge(active: false)) + unless service.update(attrs.merge(active: false)) render_api_error!('400 Bad Request', 400) end end @@ -827,17 +830,19 @@ module API TRIGGER_SERVICES.each do |service_slug, settings| helpers do + # rubocop: disable CodeReuse/ActiveRecord def slash_command_service(project, service_slug, params) project.services.active.where(template: false).find do |service| service.try(:token) == params[:token] && service.to_param == service_slug.underscore end end + # rubocop: enable CodeReuse/ActiveRecord end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 02ef89f997f..95371961398 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Settings < Grape::API before { authenticated_as_admin! } @@ -20,116 +22,105 @@ module API success Entities::ApplicationSetting end params do - optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' + given clientside_sentry_enabled: ->(val) { val } do + requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' + end + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :default_branch_protection, type: Integer, values: Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' + optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' - optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' - optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' - optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], - desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' - optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' - optional :project_export_enabled, type: Boolean, desc: 'Enable project export' - optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' - optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' - optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' - optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' - optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' - optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' - optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' - optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' - optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' given domain_blacklist_enabled: ->(val) { val } do requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' end - optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' - optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' - optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 - optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 - mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled - optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' - optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.' - optional :performance_bar_allowed_group_id, type: String, desc: 'Depreated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6 - optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6 - optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' - given require_two_factor_authentication: ->(val) { val } do - requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' - end - optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' - optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' - optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' + optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' - optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page' - optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' - given shared_runners_enabled: ->(val) { val } do - requires :shared_runners_text, type: String, desc: 'Shared runners text ' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." end + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" - optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' - optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' - optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' - requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' - requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' - requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' - requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' end - optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' - given sidekiq_throttling_enabled: ->(val) { val } do - requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' - requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' + mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled + optional :password_authentication_enabled_for_git, type: Boolean, desc: 'Flag indicating if password authentication is enabled for Git over HTTP(S)' + optional :performance_bar_allowed_group_id, type: String, desc: 'Deprecated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6 + optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.' + optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6 + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' end + optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' + optional :project_export_enabled, type: Boolean, desc: 'Enable project export' + optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' given recaptcha_enabled: ->(val) { val } do requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' end - optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' - given akismet_enabled: ->(val) { val } do - requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' end - optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for groups, projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' given sentry_enabled: ->(val) { val } do requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' end - optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' - given clientside_sentry_enabled: ->(val) { val } do - requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' - end - optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects' - optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." - optional :koding_enabled, type: Boolean, desc: 'Enable Koding' - given koding_enabled: ->(val) { val } do - requires :koding_url, type: String, desc: 'The Koding team URL' - end - optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' - given plantuml_enabled: ->(val) { val } do - requires :plantuml_url, type: String, desc: 'The PlantUML server URL' - end - optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' - optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' - optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' - optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' - given housekeeping_enabled: ->(val) { val } do - requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." - requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." - requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." - requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' end + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' - optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' - optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' - optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' - optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' + optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index 11f2b40269a..daa9598a204 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'sidekiq/api' module API diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b30305b4bc9..326d55afd0e 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API # Snippets API class Snippets < Grape::API @@ -12,7 +14,7 @@ module API end def public_snippets - SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute + SnippetsFinder.new(current_user, scope: :are_public).execute end end @@ -92,6 +94,7 @@ module API desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :content, :visibility end + # rubocop: disable CodeReuse/ActiveRecord put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet @@ -110,6 +113,7 @@ module API render_validation_error!(snippet) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Remove snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -118,6 +122,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet @@ -126,6 +131,7 @@ module API destroy_conditionally!(snippet) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a raw snippet' do detail 'This feature was introduced in GitLab 8.15.' @@ -133,14 +139,17 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/raw" do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) break not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' + header['Content-Disposition'] = 'attachment' present snippet.content end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the user agent details for a snippet' do success Entities::UserAgentDetail @@ -148,6 +157,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of a snippet' end + # rubocop: disable CodeReuse/ActiveRecord get ":id/user_agent_detail" do authenticated_as_admin! @@ -157,6 +167,7 @@ module API present snippet.user_agent_detail, with: Entities::UserAgentDetail end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb new file mode 100644 index 00000000000..72d7d994102 --- /dev/null +++ b/lib/api/submodules.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module API + class Submodules < Grape::API + before { authenticate! } + + helpers do + def commit_params(attrs) + { + submodule: attrs[:submodule], + commit_sha: attrs[:commit_sha], + branch_name: attrs[:branch], + commit_message: attrs[:commit_message] + } + end + end + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + desc 'Update existing submodule reference in repository' do + success Entities::Commit + end + params do + requires :submodule, type: String, desc: 'Url encoded full path to submodule.' + requires :commit_sha, type: String, desc: 'Commit sha to update the submodule to.' + requires :branch, type: String, desc: 'Name of the branch to commit into.' + optional :commit_message, type: String, desc: 'Commit message. If no message is provided a default one will be set.' + end + put ":id/repository/submodules/:submodule", requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + authorize! :push_code, user_project + + submodule_params = declared_params(include_missing: false) + + result = ::Submodules::UpdateService.new(user_project, current_user, commit_params(submodule_params)).execute + + if result[:status] == :success + commit_detail = user_project.repository.commit(result[:result]) + present commit_detail, with: Entities::CommitDetail + else + render_api_error!(result[:message], result[:http_status] || 400) + end + end + end + end +end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index b3e1e23031a..74ad3c35a61 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Subscriptions < Grape::API before { authenticate! } @@ -12,7 +14,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb new file mode 100644 index 00000000000..d008d1b9e97 --- /dev/null +++ b/lib/api/suggestions.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module API + class Suggestions < Grape::API + before { authenticate! } + + resource :suggestions do + desc 'Apply suggestion patch in the Merge Request it was created' do + success Entities::Suggestion + end + params do + requires :id, type: String, desc: 'The suggestion ID' + end + put ':id/apply' do + suggestion = Suggestion.find_by_id(params[:id]) + + not_found! unless suggestion + authorize! :apply_suggestion, suggestion + + result = ::Suggestions::ApplyService.new(current_user).execute(suggestion) + + if result[:status] == :success + present suggestion, with: Entities::Suggestion, current_user: current_user + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + end + end +end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index c7a460df46a..51fae0e54aa 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class SystemHooks < Grape::API include PaginationParams @@ -63,12 +65,14 @@ module API params do requires :id, type: Integer, desc: 'The ID of the system hook' end + # rubocop: disable CodeReuse/ActiveRecord delete ":id" do hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook destroy_conditionally!(hook) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 5e0afc6a7e4..f5359fd316c 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,15 +1,17 @@ +# frozen_string_literal: true + module API class Tags < Grape::API include PaginationParams - TAG_ENDPOINT_REQUIREMENTS = API::PROJECT_ENDPOINT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project repository tags' do success Entities::Tag end @@ -18,12 +20,15 @@ module API desc: 'Return tags sorted in updated by `asc` or `desc` order.' optional :order_by, type: String, values: %w[name updated], default: 'updated', desc: 'Return tags ordered by `name` or `updated` fields.' + optional :search, type: String, desc: 'Return list of tags matching the search criteria' use :pagination end get ':id/repository/tags' do - tags = ::Kaminari.paginate_array(::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}").execute) + tags = ::TagsFinder.new(user_project.repository, + sort: "#{params[:order_by]}_#{params[:sort]}", + search: params[:search]).execute - present paginate(tags), with: Entities::Tag, project: user_project + present paginate(::Kaminari.paginate_array(tags)), with: Entities::Tag, project: user_project end desc 'Get a single repository tag' do @@ -40,21 +45,35 @@ module API end desc 'Create a new repository tag' do + detail 'This optional release_description parameter was deprecated in GitLab 11.7.' success Entities::Tag end params do requires :tag_name, type: String, desc: 'The name of the tag' requires :ref, type: String, desc: 'The commit sha or branch name' optional :message, type: String, desc: 'Specifying a message creates an annotated tag' - optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end post ':id/repository/tags' do authorize_push_project result = ::Tags::CreateService.new(user_project, current_user) - .execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) + .execute(params[:tag_name], params[:ref], params[:message]) if result[:status] == :success + # Release creation with Tags API was deprecated in GitLab 11.7 + if params[:release_description].present? + release_create_params = { + tag: params[:tag_name], + name: params[:tag_name], # Name can be specified in new API + description: params[:release_description] + } + + ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute + end + present result[:tag], with: Entities::Tag, project: user_project @@ -86,44 +105,72 @@ module API end desc 'Add a release note to a tag' do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_create_release! - result = CreateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + ## + # Legacy API does not support tag auto creation. + not_found!('Tag') unless user_project.repository.find_tag(params[:tag]) + + release_create_params = { + tag: params[:tag], + name: params[:tag], # Name can be specified in new API + description: params[:description] + } + + result = ::Releases::CreateService + .new(user_project, current_user, release_create_params) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end desc "Update a tag's release note" do - success Entities::Release + detail 'This feature was deprecated in GitLab 11.7.' + success Entities::TagRelease end params do - requires :tag_name, type: String, desc: 'The name of the tag' + requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do - authorize_push_project + authorize_update_release! - result = UpdateReleaseService.new(user_project, current_user) - .execute(params[:tag_name], params[:description]) + result = ::Releases::UpdateService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::TagRelease else render_api_error!(result[:message], result[:http_status]) end end end + + helpers do + def authorize_create_release! + authorize! :create_release, user_project + end + + def authorize_update_release! + authorize! :update_release, release + end + + def release + @release ||= user_project.releases.find_by_tag(params[:tag]) + end + end end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 41862768a3f..51f357d9477 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,46 +1,22 @@ +# frozen_string_literal: true + module API class Templates < Grape::API include PaginationParams GLOBAL_TEMPLATE_TYPES = { gitignores: { - klass: Gitlab::Template::GitignoreTemplate, gitlab_version: 8.8 }, gitlab_ci_ymls: { - klass: Gitlab::Template::GitlabCiYmlTemplate, gitlab_version: 8.9 }, dockerfiles: { - klass: Gitlab::Template::DockerfileTemplate, gitlab_version: 8.15 } }.freeze - PROJECT_TEMPLATE_REGEX = - %r{[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]}xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - %r{[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]}xi.freeze helpers do - def parsed_license_template - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - template = Licensee::License.new(params[:name]) - - template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - template - end - def render_response(template_type, template) not_found!(template_type.to_s.singularize) unless template present template, with: Entities::Template @@ -56,11 +32,12 @@ module API use :pagination end get "templates/licenses" do - options = { - featured: declared(params)[:popular].present? ? true : nil - } - licences = ::Kaminari.paginate_array(Licensee::License.all(options)) - present paginate(licences), with: Entities::License + popular = declared(params)[:popular] + popular = to_boolean(popular) if popular.present? + + templates = TemplateFinder.build(:licenses, nil, popular: popular).execute + + present paginate(::Kaminari.paginate_array(templates)), with: ::API::Entities::License end desc 'Get the text for a specific license' do @@ -71,15 +48,19 @@ module API requires :name, type: String, desc: 'The name of the template' end get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params)[:name]) + template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute + + not_found!('License') unless template.present? - template = parsed_license_template + template.resolve!( + project_name: params[:project].presence, + fullname: params[:fullname].presence || current_user&.name + ) present template, with: ::API::Entities::License end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| - klass = properties[:klass] gitlab_version = properties[:gitlab_version] desc 'Get the list of the available template' do @@ -90,7 +71,7 @@ module API use :pagination end get "templates/#{template_type}" do - templates = ::Kaminari.paginate_array(klass.all) + templates = ::Kaminari.paginate_array(TemplateFinder.build(template_type, nil).execute) present paginate(templates), with: Entities::TemplatesList end @@ -101,8 +82,9 @@ module API params do requires :name, type: String, desc: 'The name of the template' end - get "templates/#{template_type}/:name" do - new_template = klass.find(declared(params)[:name]) + get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ } do + finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name]) + new_template = finder.execute render_response(template_type, new_template) end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 2bb451dea89..93fe06bec27 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API module TimeTrackingEndpoints extend ActiveSupport::Concern diff --git a/lib/api/todos.rb b/lib/api/todos.rb index c6dbcf84e3a..64ac8ece56c 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Todos < Grape::API include PaginationParams @@ -12,7 +14,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_iid".to_sym diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b29e660c6e0..8fc7c7361e1 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Triggers < Grape::API include PaginationParams @@ -5,12 +7,12 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Trigger a GitLab project pipeline' do success Entities::Pipeline end params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false requires :token, type: String, desc: 'The unique token of trigger' optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end @@ -42,20 +44,22 @@ module API params do use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/triggers' do authenticate! authorize! :admin_build, user_project triggers = user_project.triggers.includes(:trigger_requests) - present paginate(triggers), with: Entities::Trigger + present paginate(triggers), with: Entities::Trigger, current_user: current_user end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get specific trigger of a project' do success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end get ':id/triggers/:trigger_id' do authenticate! @@ -64,14 +68,14 @@ module API trigger = user_project.triggers.find(params.delete(:trigger_id)) break not_found!('Trigger') unless trigger - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user end desc 'Create a trigger' do success Entities::Trigger end params do - requires :description, type: String, desc: 'The trigger description' + requires :description, type: String, desc: 'The trigger description' end post ':id/triggers' do authenticate! @@ -81,7 +85,7 @@ module API declared_params(include_missing: false).merge(owner: current_user)) if trigger.valid? - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -102,7 +106,7 @@ module API break not_found!('Trigger') unless trigger if trigger.update(declared_params(include_missing: false)) - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -112,7 +116,7 @@ module API success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end post ':id/triggers/:trigger_id/take_ownership' do authenticate! @@ -123,7 +127,7 @@ module API if trigger.update(owner: current_user) status :ok - present trigger, with: Entities::Trigger + present trigger, with: Entities::Trigger, current_user: current_user else render_validation_error!(trigger) end @@ -133,7 +137,7 @@ module API success Entities::Trigger end params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end delete ':id/triggers/:trigger_id' do authenticate! diff --git a/lib/api/users.rb b/lib/api/users.rb index 14b8a796c8e..8ce09a8881b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Users < Grape::API include PaginationParams @@ -14,11 +16,14 @@ module API end helpers do + # rubocop: disable CodeReuse/ActiveRecord def find_user_by_id(params) id = params[:user_id] || params[:id] User.find_by(id: id) || not_found!('User') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def reorder_users(users) if params[:order_by] && params[:sort] users.reorder(params[:order_by] => params[:sort]) @@ -26,6 +31,7 @@ module API users end end + # rubocop: enable CodeReuse/ActiveRecord params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' @@ -38,10 +44,12 @@ module API optional :provider, type: String, desc: 'The external provider' optional :bio, type: String, desc: 'The biography of the user' optional :location, type: String, desc: 'The location of the user' + optional :public_email, type: String, desc: 'The public email of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' optional :avatar, type: File, desc: 'Avatar image for user' + optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' all_or_none_of :extern_uid, :provider end @@ -73,6 +81,7 @@ module API use :pagination use :with_custom_attributes end + # rubocop: disable CodeReuse/ActiveRecord get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) @@ -96,10 +105,11 @@ module API entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin - users, options = with_custom_attributes(users, with: entity) + users, options = with_custom_attributes(users, { with: entity, current_user: current_user }) present paginate(users), options end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a single user' do success Entities::User @@ -109,15 +119,28 @@ module API use :with_custom_attributes end + # rubocop: disable CodeReuse/ActiveRecord get ":id" do user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) - opts = current_user&.admin? ? { with: Entities::UserWithAdmin } : { with: Entities::User } + opts = { with: current_user&.admin? ? Entities::UserWithAdmin : Entities::User, current_user: current_user } user, opts = with_custom_attributes(user, opts) present user, opts end + # rubocop: enable CodeReuse/ActiveRecord + + desc "Get the status of a user" + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + end + get ":user_id/status", requirements: API::USER_REQUIREMENTS do + user = find_user(params[:user_id]) + not_found!('User') unless user && can?(current_user, :read_user, user) + + present user.status || {}, with: Entities::UserStatus + end desc 'Create a user. Available only for admins.' do success Entities::UserPublic @@ -139,15 +162,15 @@ module API user = ::Users::CreateService.new(current_user, params).execute(skip_authorization: true) if user.persisted? - present user, with: Entities::UserPublic + present user, with: Entities::UserPublic, current_user: current_user else conflict!('Email has already been taken') if User - .where(email: user.email) - .count > 0 + .by_any_email(user.email.downcase) + .any? conflict!('Username has already been taken') if User - .where(username: user.username) - .count > 0 + .by_username(user.username) + .any? render_validation_error!(user) end @@ -165,6 +188,7 @@ module API optional :username, type: String, desc: 'The username of the user' use :optional_attributes end + # rubocop: disable CodeReuse/ActiveRecord put ":id" do authenticated_as_admin! @@ -172,11 +196,11 @@ module API not_found!('User') unless user conflict!('Email has already been taken') if params[:email] && - User.where(email: params[:email]) + User.by_any_email(params[:email].downcase) .where.not(id: user.id).count > 0 conflict!('Username has already been taken') if params[:username] && - User.where(username: params[:username]) + User.by_username(params[:username]) .where.not(id: user.id).count > 0 user_params = declared_params(include_missing: false) @@ -186,7 +210,7 @@ module API identity = user.identities.find_by(provider: identity_attrs[:provider]) if identity - identity.update_attributes(identity_attrs) + identity.update(identity_attrs) else identity = user.identities.build(identity_attrs) identity.save @@ -198,11 +222,12 @@ module API result = ::Users::UpdateService.new(current_user, user_params.except(:extern_uid, :provider).merge(user: user)).execute if result[:status] == :success - present user, with: Entities::UserPublic + present user, with: Entities::UserPublic, current_user: current_user else render_validation_error!(user) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add an SSH key to a specified user. Available only for admins.' do success Entities::SSHKey @@ -212,6 +237,7 @@ module API requires :key, type: String, desc: 'The new SSH key' requires :title, type: String, desc: 'The title of the new SSH key' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/keys" do authenticated_as_admin! @@ -226,22 +252,23 @@ module API render_validation_error!(key) end end + # rubocop: enable CodeReuse/ActiveRecord - desc 'Get the SSH keys of a specified user. Available only for admins.' do + desc 'Get the SSH keys of a specified user.' do success Entities::SSHKey end params do requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/keys' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) - not_found!('User') unless user + not_found!('User') unless user && can?(current_user, :read_user, user) present paginate(user.keys), with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing SSH key from a specified user. Available only for admins.' do success Entities::SSHKey @@ -250,6 +277,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/keys/:key_id' do authenticated_as_admin! @@ -261,6 +289,7 @@ module API destroy_conditionally!(key) end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a GPG key to a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -270,6 +299,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key, type: String, desc: 'The new GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/gpg_keys' do authenticated_as_admin! @@ -284,6 +314,7 @@ module API render_validation_error!(key) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the GPG keys of a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -293,6 +324,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/gpg_keys' do authenticated_as_admin! @@ -301,6 +333,7 @@ module API present paginate(user.gpg_keys), with: Entities::GPGKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing GPG key from a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -309,6 +342,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/gpg_keys/:key_id' do authenticated_as_admin! @@ -321,6 +355,7 @@ module API status 204 key.destroy end + # rubocop: enable CodeReuse/ActiveRecord desc 'Revokes an existing GPG key from a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' @@ -329,6 +364,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/gpg_keys/:key_id/revoke' do authenticated_as_admin! @@ -341,6 +377,7 @@ module API key.revoke status :accepted end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add an email address to a specified user. Available only for admins.' do success Entities::Email @@ -348,7 +385,9 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' requires :email, type: String, desc: 'The email of the user' + optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified' end + # rubocop: disable CodeReuse/ActiveRecord post ":id/emails" do authenticated_as_admin! @@ -363,6 +402,7 @@ module API render_validation_error!(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get the emails addresses of a specified user. Available only for admins.' do success Entities::Email @@ -371,6 +411,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get ':id/emails' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -378,6 +419,7 @@ module API present paginate(user.emails), with: Entities::Email end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an email address of a specified user. Available only for admins.' do success Entities::Email @@ -386,6 +428,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/emails/:email_id' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -398,6 +441,7 @@ module API Emails::DestroyService.new(current_user, user: user).execute(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a user. Available only for admins.' do success Entities::Email @@ -406,6 +450,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end + # rubocop: disable CodeReuse/ActiveRecord delete ":id" do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279') @@ -418,11 +463,13 @@ module API user.delete_async(deleted_by: current_user, params: params) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Block a user. Available only for admins.' params do requires :id, type: Integer, desc: 'The ID of the user' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/block' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -434,11 +481,13 @@ module API forbidden!('LDAP blocked users cannot be modified by the API') end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Unblock a user. Available only for admins.' params do requires :id, type: Integer, desc: 'The ID of the user' end + # rubocop: disable CodeReuse/ActiveRecord post ':id/unblock' do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -450,6 +499,7 @@ module API user.activate end end + # rubocop: enable CodeReuse/ActiveRecord params do requires :user_id, type: Integer, desc: 'The ID of the user' @@ -463,7 +513,7 @@ module API end def find_impersonation_token - finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') + finder.find_by_id(declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') end end @@ -481,7 +531,7 @@ module API desc 'Create a impersonation token. Available only for admins.' do detail 'This feature was introduced in GitLab 9.0' - success Entities::ImpersonationToken + success Entities::ImpersonationTokenWithToken end params do requires :name, type: String, desc: 'The name of the impersonation token' @@ -492,7 +542,7 @@ module API impersonation_token = finder.build(declared_params(include_missing: false)) if impersonation_token.save - present impersonation_token, with: Entities::ImpersonationToken + present impersonation_token, with: Entities::ImpersonationTokenWithToken else render_validation_error!(impersonation_token) end @@ -531,18 +581,22 @@ module API authenticate! end - desc 'Get the currently authenticated user' do - success Entities::UserPublic - end - get do - entity = - if current_user.admin? - Entities::UserWithAdmin - else - Entities::UserPublic - end + # Enabling /user endpoint for the v3 version to allow oauth + # authentication through this endpoint. + version %w(v3 v4), using: :path do + desc 'Get the currently authenticated user' do + success Entities::UserPublic + end + get do + entity = + if current_user.admin? + Entities::UserWithAdmin + else + Entities::UserPublic + end - present current_user, with: entity + present current_user, with: entity, current_user: current_user + end end desc "Get the currently authenticated user's SSH keys" do @@ -561,12 +615,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord get "keys/:key_id" do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key present key, with: Entities::SSHKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a new SSH key to the currently authenticated user' do success Entities::SSHKey @@ -591,12 +647,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete "keys/:key_id" do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key destroy_conditionally!(key) end + # rubocop: enable CodeReuse/ActiveRecord desc "Get the currently authenticated user's GPG keys" do detail 'This feature was added in GitLab 10.0' @@ -616,12 +674,14 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord get 'gpg_keys/:key_id' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key present key, with: Entities::GPGKey end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add a new GPG key to the currently authenticated user' do detail 'This feature was added in GitLab 10.0' @@ -646,6 +706,7 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the GPG key' end + # rubocop: disable CodeReuse/ActiveRecord post 'gpg_keys/:key_id/revoke' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -653,6 +714,7 @@ module API key.revoke status :accepted end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete a GPG key from the currently authenticated user' do detail 'This feature was added in GitLab 10.0' @@ -660,6 +722,7 @@ module API params do requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + # rubocop: disable CodeReuse/ActiveRecord delete 'gpg_keys/:key_id' do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -667,6 +730,7 @@ module API status 204 key.destroy end + # rubocop: enable CodeReuse/ActiveRecord desc "Get the currently authenticated user's email addresses" do success Entities::Email @@ -684,12 +748,14 @@ module API params do requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord get "emails/:email_id" do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email present email, with: Entities::Email end + # rubocop: enable CodeReuse/ActiveRecord desc 'Add new email address to the currently authenticated user' do success Entities::Email @@ -711,6 +777,7 @@ module API params do requires :email_id, type: Integer, desc: 'The ID of the email' end + # rubocop: disable CodeReuse/ActiveRecord delete "emails/:email_id" do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -719,12 +786,14 @@ module API Emails::DestroyService.new(current_user, user: current_user).execute(email) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Get a list of user activities' params do optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' use :pagination end + # rubocop: disable CodeReuse/ActiveRecord get "activities" do authenticated_as_admin! @@ -734,6 +803,31 @@ module API present paginate(activities), with: Entities::UserActivity end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Set the status of the current user' do + success Entities::UserStatus + end + params do + optional :emoji, type: String, desc: "The emoji to set on the status" + optional :message, type: String, desc: "The status message to set" + end + put "status" do + forbidden! unless can?(current_user, :update_user_status, current_user) + + if ::Users::SetStatusService.new(current_user, declared_params).execute + present current_user.status, with: Entities::UserStatus + else + render_validation_error!(current_user.status) + end + end + + desc 'get the status of the current user' do + success Entities::UserStatus + end + get 'status' do + present current_user.status || {}, with: Entities::UserStatus + end end end end diff --git a/lib/api/validations/types/safe_file.rb b/lib/api/validations/types/safe_file.rb new file mode 100644 index 00000000000..53b5790bfa2 --- /dev/null +++ b/lib/api/validations/types/safe_file.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# This module overrides the Grape type validator defined in +# https://github.com/ruby-grape/grape/blob/master/lib/grape/validations/types/file.rb +module API + module Validations + module Types + class SafeFile < ::Grape::Validations::Types::File + def value_coerced?(value) + super && value[:tempfile].is_a?(Tempfile) + end + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index a34de9410e8..148deb86c4c 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Variables < Grape::API include PaginationParams @@ -9,7 +11,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get project variables' do success Entities::Variable end @@ -27,6 +29,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do key = params[:key] variable = user_project.variables.find_by(key: key) @@ -35,6 +38,7 @@ module API present variable, with: Entities::Variable end + # rubocop: enable CodeReuse/ActiveRecord desc 'Create a new variable in a project' do success Entities::Variable @@ -64,6 +68,7 @@ module API optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' end + # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) @@ -77,6 +82,7 @@ module API render_validation_error!(variable) end end + # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing variable from a project' do success Entities::Variable @@ -84,6 +90,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end + # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) not_found!('Variable') unless variable @@ -92,6 +99,7 @@ module API status 204 variable.destroy end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/version.rb b/lib/api/version.rb index 3b10bfa6a7d..74cd857f447 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module API class Version < Grape::API before { authenticate! } diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index b3fc4e876ad..ef0e3decc2c 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -1,6 +1,16 @@ +# frozen_string_literal: true + module API class Wikis < Grape::API helpers do + def commit_params(attrs) + { + file_name: attrs[:file][:filename], + file_content: attrs[:file][:tempfile].read, + branch_name: attrs[:branch] + } + end + params :wiki_page_params do requires :content, type: String, desc: 'Content of a wiki page' requires :title, type: String, desc: 'Title of a wiki page' @@ -12,7 +22,9 @@ module API end end - resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + WIKI_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(slug: API::NO_SLASH_URL_PART_REGEX) + + resource :projects, requirements: WIKI_ENDPOINT_REQUIREMENTS do desc 'Get a list of wiki pages' do success Entities::WikiPageBasic end @@ -84,6 +96,29 @@ module API status 204 WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page) end + + desc 'Upload an attachment to the wiki repository' do + detail 'This feature was introduced in GitLab 11.3.' + success Entities::WikiAttachment + end + params do + requires :file, type: ::API::Validations::Types::SafeFile, desc: 'The attachment file to be uploaded' + optional :branch, type: String, desc: 'The name of the branch' + end + post ":id/wikis/attachments" do + authorize! :create_wiki, user_project + + result = ::Wikis::CreateAttachmentService.new(user_project, + current_user, + commit_params(declared_params(include_missing: false))).execute + + if result[:status] == :success + status(201) + present OpenStruct.new(result[:result]), with: Entities::WikiAttachment + else + render_api_error!(result[:message], 400) + end + end end end end diff --git a/lib/backup.rb b/lib/backup.rb new file mode 100644 index 00000000000..2712b33b4b4 --- /dev/null +++ b/lib/backup.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module Backup + Error = Class.new(StandardError) +end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 45a935ab352..33658ae225f 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index adf85ca4719..5e795a449de 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 1608f7ad02d..e6bf3d1856f 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' module Backup @@ -44,7 +46,7 @@ module Backup end report_success(success) - abort 'Backup failed' unless success + raise Backup::Error, 'Backup failed' unless success end def restore @@ -72,7 +74,7 @@ module Backup end report_success(success) - abort 'Restore failed' unless success + abort Backup::Error, 'Restore failed' unless success end protected diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 9895db9e451..2bac84846c5 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'open3' require_relative 'helper' @@ -26,20 +28,29 @@ module Backup unless status.zero? puts output - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end - run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(#{tar} --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(#{tar} --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end end def restore backup_existing_files_dir - run_pipeline!([%w(gzip -cd), %W(tar --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) + end + + def tar + if system(*%w[gtar --version], out: '/dev/null') + # It looks like we can get GNU tar by running 'gtar' + 'gtar' + else + 'tar' + end end def backup_existing_files_dir @@ -60,8 +71,14 @@ module Backup end def run_pipeline!(cmd_list, options = {}) - status_list = Open3.pipeline(*cmd_list, options) - abort 'Backup failed' unless status_list.compact.all?(&:success?) + err_r, err_w = IO.pipe + options[:err] = err_w + status = Open3.pipeline(*cmd_list, options) + err_w.close + return if status.compact.all?(&:success?) + + regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/ + raise Backup::Error, 'Backup failed' unless err_r.read =~ regex end end end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb index 54b9ce10b4d..22f00aef569 100644 --- a/lib/backup/helper.rb +++ b/lib/backup/helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup module Helper def access_denied_error(path) diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 185ff8ae6bd..0dfe56e214f 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index a8da0c7edef..06b0338b1ed 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Backup class Manager ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze @@ -27,7 +29,7 @@ module Backup progress.puts "done".color(:green) else puts "creating archive #{tar_file} failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end upload @@ -48,11 +50,12 @@ module Backup if directory.files.create(key: remote_target, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, encryption: Gitlab.config.backup.upload.encryption, + encryption_key: Gitlab.config.backup.upload.encryption_key, storage_class: Gitlab.config.backup.upload.storage_class) progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end end @@ -66,7 +69,7 @@ module Backup progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) - abort 'Backup failed' + raise Backup::Error, 'Backup failed' end end end @@ -162,7 +165,7 @@ module Backup def tar_version tar_version, _ = Gitlab::Popen.popen(%w(tar --version)) - tar_version.force_encoding('locale').split("\n").first + tar_version.dup.force_encoding('locale').split("\n").first end def skipped?(item) @@ -193,7 +196,7 @@ module Backup if connection.service == ::Fog::Storage::Local connection.directories.create(key: remote_directory) else - connection.directories.get(remote_directory) + connection.directories.new(key: remote_directory) end end @@ -241,6 +244,7 @@ module Backup backup_created_at: Time.now, gitlab_version: Gitlab::VERSION, tar_version: tar_version, + installation_type: Gitlab::INSTALLATION_TYPE, skipped: ENV["SKIP"] } end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index 542e35a7c7c..a4be728df08 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index 35821805797..d16ed2facf1 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 84670d6582e..184c7418e75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,11 +1,10 @@ +# frozen_string_literal: true + require 'yaml' -require_relative 'helper' module Backup class Repository - include Backup::Helper - # rubocop:disable Metrics/AbcSize - + include Gitlab::ShellAdapter attr_reader :progress def initialize(progress) @@ -17,113 +16,66 @@ module Backup Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = path_to_repo(project) - path_to_project_bundle = path_to_bundle(project) - # Create namespace dir or hashed path if missing if project.hashed_storage?(:repository) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) else FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace end - if empty_repo?(project) - progress.puts "[SKIPPED]".color(:cyan) + if !empty_repo?(project) + backup_project(project) + progress.puts "[DONE]".color(:green) else - in_path(path_to_project_repo) do |dir| - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end + progress.puts "[SKIPPED]".color(:cyan) end wiki = ProjectWiki.new(project) - path_to_wiki_repo = path_to_repo(wiki) - path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_wiki_repo) - progress.print " * #{display_repo_path(wiki)} ... " - - if empty_repo?(wiki) - progress.puts " [SKIPPED]".color(:cyan) - else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(wiki, cmd.join(' '), output) - end - end + if !empty_repo?(wiki) + backup_project(wiki) + progress.puts "[DONE] Wiki".color(:green) + else + progress.puts "[SKIPPED] Wiki".color(:cyan) end end end def prepare_directories - Gitlab.config.repositories.storages.each do |name, repository_storage| - delete_all_repositories(name, repository_storage) + Gitlab.config.repositories.storages.each do |name, _repository_storage| + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories end end - def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories - else - local_delete_all_repositories(name, repository_storage) - end - end - end + def backup_project(project) + path_to_project_bundle = path_to_bundle(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .create_bundle(path_to_project_bundle) - def local_delete_all_repositories(name, repository_storage) - path = repository_storage.legacy_disk_path - return unless File.exist?(path) + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') + end - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] + def backup_custom_hooks(project) + FileUtils.mkdir_p(project_backup_path(project)) - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) - end + custom_hooks_path = custom_hooks_tar(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .backup_custom_hooks(custom_hooks_path) end def restore_custom_hooks(project) - # TODO: Need to find a way to do this for gitaly - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195 - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = path_to_repo(project) - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end + return unless Dir.exist?(project_backup_path(project)) + return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? + + custom_hooks_path = custom_hooks_tar(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .restore_custom_hooks(custom_hooks_path) end def restore prepare_directories - gitlab_shell = Gitlab::Shell.new Project.find_each(batch_size: 1000) do |project| progress.print " * #{project.full_path} ... " @@ -133,7 +85,8 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - project.repository.create_from_bundle path_to_project_bundle + project.repository.create_from_bundle(path_to_project_bundle) + restore_custom_hooks(project) restore_repo_success = true rescue => e restore_repo_success = false @@ -149,8 +102,6 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - restore_custom_hooks(project) - wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) @@ -158,6 +109,8 @@ module Backup progress.print " * #{wiki.full_path} ... " begin wiki.repository.create_from_bundle(path_to_wiki_bundle) + restore_custom_hooks(wiki) + progress.puts "[DONE]".color(:green) rescue => e progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) @@ -165,55 +118,34 @@ module Backup end end end + + restore_object_pools end - # rubocop:enable Metrics/AbcSize protected - def path_to_repo(project) - project.repository.path_to_repo - end - def path_to_bundle(project) File.join(backup_repos_path, project.disk_path + '.bundle') end - def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.disk_path) + def project_backup_path(project) + File.join(backup_repos_path, project.disk_path) + end - if dir - File.join(path, "#{dir}.tar") - else - path - end + def custom_hooks_tar(project) + File.join(project_backup_path(project), "custom_hooks.tar") end def backup_repos_path File.join(Gitlab.config.backup.path, 'repositories') end - def in_path(path) - return unless Dir.exist?(path) - - dir_entries = Dir.entries(path) - - if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') - yield('custom_hooks') - end - end - def prepare FileUtils.rm_rf(backup_repos_path) - # Ensure the parent dir of backup_repos_path exists FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_repos_path before us FileUtils.mkdir(backup_repos_path, mode: 0700) end - def silent - { err: '/dev/null', out: '/dev/null' } - end - private def progress_warn(project, cmd, output) @@ -222,23 +154,24 @@ module Backup end def empty_repo?(project_or_wiki) - # Protect against stale caches project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.empty? end - def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound, GRPC::BadStatus => e - raise Error, e + def restore_object_pools + PoolRepository.includes(:source_project).find_each do |pool| + progress.puts " - Object pool #{pool.disk_path}..." + + pool.source_project ||= pool.member_projects.first.root_of_fork_network + pool.state = 'none' + pool.save + + pool.schedule + end end end end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 49b117a7ee3..9577df2634a 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'backup/files' module Backup diff --git a/lib/banzai.rb b/lib/banzai.rb index 5df98f66f3b..1eb41ff7133 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -1,4 +1,13 @@ +# frozen_string_literal: true + module Banzai + # if you need to render markdown, then you probably need to post_process as well, + # such as removing references that the current user doesn't have + # permission to make + def self.render_and_post_process(text, context = {}) + post_process(render(text, context), context) + end + def self.render(text, context = {}) Renderer.render(text, context) end diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index 355c364b07b..6d01d51955c 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ColorParser ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 diff --git a/lib/banzai/commit_renderer.rb b/lib/banzai/commit_renderer.rb index c351a155ae5..f346151a3c1 100644 --- a/lib/banzai/commit_renderer.rb +++ b/lib/banzai/commit_renderer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module CommitRenderer ATTRIBUTES = [:description, :title].freeze diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index 3f1e95d4cc0..b7344808989 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai # Common methods for ReferenceFilters that support an optional cross-project # reference. @@ -13,6 +15,7 @@ module Banzai # Returns a Project, or nil if the reference can't be found def parent_from_ref(ref) return context[:project] || context[:group] unless ref + return context[:project] if context[:project]&.full_path == ref Project.find_by_full_path(ref) end diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index 3eb544dfef9..7d9766c906c 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter def self.[](name) diff --git a/lib/banzai/filter/absolute_link_filter.rb b/lib/banzai/filter/absolute_link_filter.rb index 1ec6201523f..a9bdb004c4b 100644 --- a/lib/banzai/filter/absolute_link_filter.rb +++ b/lib/banzai/filter/absolute_link_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'uri' module Banzai @@ -27,6 +29,7 @@ module Banzai end def absolute_link_attr(uri) + # Here we really want to expand relative path to absolute path URI.join(Gitlab.config.gitlab.url, uri).to_s end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 60a12dca9d3..4764f8e1e19 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # Issues, Merge Requests, Snippets, Commits and Commit Ranges share @@ -100,6 +102,11 @@ module Banzai ref_pattern = object_class.reference_pattern link_pattern = object_class.link_reference_pattern + # Compile often used regexps only once outside of the loop + ref_pattern_anchor = /\A#{ref_pattern}\z/ + link_pattern_start = /\A#{link_pattern}/ + link_pattern_anchor = /\A#{link_pattern}\z/ + nodes.each do |node| if text_node?(node) && ref_pattern replace_text_when_pattern_matches(node, ref_pattern) do |content| @@ -108,7 +115,7 @@ module Banzai elsif element_node?(node) yield_valid_link(node) do |link, inner_html| - if ref_pattern && link =~ /\A#{ref_pattern}\z/ + if ref_pattern && link =~ ref_pattern_anchor replace_link_node_with_href(node, link) do object_link_filter(link, ref_pattern, link_content: inner_html) end @@ -118,7 +125,7 @@ module Banzai next unless link_pattern - if link == inner_html && inner_html =~ /\A#{link_pattern}/ + if link == inner_html && inner_html =~ link_pattern_start replace_link_node_with_text(node, link) do object_link_filter(inner_html, link_pattern, link_reference: true) end @@ -126,7 +133,7 @@ module Banzai next end - if link =~ /\A#{link_pattern}\z/ + if link =~ link_pattern_anchor replace_link_node_with_href(node, link) do object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true) end @@ -289,7 +296,7 @@ module Banzai # Returns projects for the given paths. def find_for_paths(paths) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? cache = refs_cache to_query = paths - cache.keys @@ -333,7 +340,7 @@ module Banzai end def refs_cache - RequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end def parent_type diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb index c9fcf057c5f..88439f06b5f 100644 --- a/lib/banzai/filter/ascii_doc_post_processing_filter.rb +++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter class AsciiDocPostProcessingFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 4a143baeef6..f3061bad4ff 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'uri' module Banzai @@ -6,6 +8,10 @@ module Banzai # # Based on HTML::Pipeline::AutolinkFilter # + # Note that our CommonMark parser, `commonmarker` (using the autolink extension) + # handles standard autolinking, like http/https. We detect additional + # schemes (smb, rdar, etc). + # # Context options: # :autolink - Boolean, skips all processing done by this filter when false # :link_attr - Hash of attributes for the generated links @@ -105,10 +111,13 @@ module Banzai end end - # match has come from node.to_html above, so we know it's encoded - # correctly. + # Since this came from a Text node, make sure the new href is encoded. + # `commonmarker` percent encodes the domains of links it handles, so + # do the same (instead of using `normalized_encode`). + href_safe = Addressable::URI.encode(match).html_safe + html_safe_match = match.html_safe - options = link_options.merge(href: html_safe_match) + options = link_options.merge(href: href_safe) content_tag(:a, html_safe_match, options) + dropped end diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index d2c4b1e4d76..ad367cc5efe 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -1,28 +1,10 @@ +# frozen_string_literal: true + module Banzai module Filter class BlockquoteFenceFilter < HTML::Pipeline::TextFilter REGEX = %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `>>>` blocks which are ignored by this filter - # ``` - - ^``` - .+? - \n```$ - ) - | - (?<html> - # HTML block: - # <tag> - # Anything, including `>>>` blocks which are ignored by this filter - # </tag> - - ^<[^>]+?>\n - .+? - \n<\/[^>]+?>$ - ) + #{::Gitlab::Regex.markdown_code_or_html_blocks} | (?: # Blockquote: @@ -30,14 +12,14 @@ module Banzai # Anything, including code and HTML blocks # >>> - ^>>>\n + ^>>>\ *\n (?<quote> (?: # Any character that doesn't introduce a code or HTML block (?! ^``` | - ^<[^>]+?>\n + ^<[^>]+?>\ *\n ) . | @@ -48,7 +30,7 @@ module Banzai \g<html> )+? ) - \n>>>$ + \n>>>\ *$ ) }mx.freeze diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb index 6ab29ac281f..6d9bdb9cbd3 100644 --- a/lib/banzai/filter/color_filter.rb +++ b/lib/banzai/filter/color_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that renders `color` followed by a color "chip". diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 01b3b0dafb9..d6b46236a49 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces commit range references with links. diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 8cd92a1adba..c3e5ac41cb8 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces commit references with links. diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb index 7b55e8b36f6..f49c4b403db 100644 --- a/lib/banzai/filter/commit_trailers_filter.rb +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces users' names and emails in commit trailers diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index e1261e7bbbe..fa1690f73ad 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,12 +1,11 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/emoji.js module Banzai module Filter # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb index 265924abe24..26bcf5c04b4 100644 --- a/lib/banzai/filter/epic_reference_filter.rb +++ b/lib/banzai/filter/epic_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # The actual filter is implemented in the EE mixin @@ -7,6 +9,12 @@ module Banzai def self.object_class Epic end + + private + + def group + context[:group] || context[:project]&.group + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index ed01a72ff9f..8159dcfed72 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces external issue tracker references with links. @@ -95,9 +97,7 @@ module Banzai private def external_issues_cached(attribute) - return project.public_send(attribute) unless RequestStore.active? # rubocop:disable GitlabSecurity/PublicSend - - cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend cached_attributes[project.id][attribute] end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index d6327ef31cb..61ee3eac216 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,19 +1,32 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter - SCHEMES = ['http', 'https', nil].freeze + SCHEMES = ['http', 'https', nil].freeze + RTLO = "\u202E".freeze + ENCODED_RTLO = '%E2%80%AE'.freeze def call links.each do |node| - uri = uri(node['href'].to_s) - next unless uri - - node.set_attribute('href', uri.to_s) + # URI.parse does stricter checking on the url than Addressable, + # such as on `mailto:` links. Since we've been using it, do an + # initial parse for validity and then use Addressable + # for IDN support, etc + uri = uri_strict(node['href'].to_s) + if uri + node.set_attribute('href', uri.to_s) + addressable_uri = addressable_uri(node['href']) + else + addressable_uri = nil + end - if SCHEMES.include?(uri.scheme) && external_url?(uri) - node.set_attribute('rel', 'nofollow noreferrer noopener') - node.set_attribute('target', '_blank') + unless internal_url?(addressable_uri) + punycode_autolink_node!(addressable_uri, node) + sanitize_link_text!(node) + add_malicious_tooltip!(addressable_uri, node) + add_nofollow!(addressable_uri, node) end end @@ -22,27 +35,85 @@ module Banzai private - def uri(href) + def uri_strict(href) URI.parse(href) rescue URI::Error nil end + def addressable_uri(href) + Addressable::URI.parse(href) + rescue Addressable::URI::InvalidURIError + nil + end + def links query = 'descendant-or-self::a[@href and not(@href = "")]' doc.xpath(query) end - def external_url?(uri) + def internal_url?(uri) + return false if uri.nil? # Relative URLs miss a hostname - return false unless uri.hostname + return true unless uri.hostname - uri.hostname != internal_url.hostname + uri.hostname == internal_url.hostname end def internal_url @internal_url ||= URI.parse(Gitlab.config.gitlab.url) end + + # Only replace an autolink with an IDN with it's punycode + # version if we need emailable links. Otherwise let it + # be shown normally and the tooltips will show the + # punycode version. + def punycode_autolink_node!(uri, node) + return unless uri + return unless context[:emailable_links] + + unencoded_uri_str = Addressable::URI.unencode(node['href']) + + if unencoded_uri_str == node.content && idn?(uri) + node.content = uri.normalize + end + end + + # escape any right-to-left (RTLO) characters in link text + def sanitize_link_text!(node) + node.inner_html = node.inner_html.gsub(RTLO, ENCODED_RTLO) + end + + # If the domain is an international domain name (IDN), + # let's expose with a tooltip in case it's intended + # to be malicious. This is particularly useful for links + # where the link text is not the same as the actual link. + # We will continue to show the unicode version of the domain + # in autolinked link text, which could contain emojis, etc. + # + # Also show the tooltip if the url contains the RTLO character, + # as this is an indicator of a malicious link + def add_malicious_tooltip!(uri, node) + if idn?(uri) || has_encoded_rtlo?(uri) + node.add_class('has-tooltip') + node.set_attribute('title', uri.normalize) + end + end + + def add_nofollow!(uri, node) + if SCHEMES.include?(uri&.scheme) + node.set_attribute('rel', 'nofollow noreferrer noopener') + node.set_attribute('target', '_blank') + end + end + + def idn?(uri) + uri&.normalized_host&.start_with?('xn--') + end + + def has_encoded_rtlo?(uri) + uri&.to_s&.include?(ENCODED_RTLO) + end end end end diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb new file mode 100644 index 00000000000..97527976437 --- /dev/null +++ b/lib/banzai/filter/footnote_filter.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML Filter for footnotes + # + # Footnotes are supported in CommonMark. However we were stripping + # the ids during sanitization. Those are now allowed. + # + # Footnotes are numbered the same - the first one has `id=fn1`, the + # second is `id=fn2`, etc. In order to allow footnotes when rendering + # multiple markdown blocks on a page, we need to make each footnote + # reference unique. + # + # This filter adds a random number to each footnote (the same number + # can be used for a single render). So you get `id=fn1-4335` and `id=fn2-4335`. + # + class FootnoteFilter < HTML::Pipeline::Filter + INTEGER_PATTERN = /\A\d+\z/.freeze + FOOTNOTE_ID_PREFIX = 'fn'.freeze + FOOTNOTE_LINK_ID_PREFIX = 'fnref'.freeze + FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}\d+\z/.freeze + FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}\d+\z/.freeze + FOOTNOTE_START_NUMBER = 1 + + def call + return doc unless first_footnote = doc.at_css("ol > li[id=#{fn_id(FOOTNOTE_START_NUMBER)}]") + + # Sanitization stripped off the section wrapper - add it back in + first_footnote.parent.wrap('<section class="footnotes">') + rand_suffix = "-#{random_number}" + + doc.css('sup > a[id]').each do |link_node| + ref_num = link_node[:id].delete_prefix(FOOTNOTE_LINK_ID_PREFIX) + footnote_node = doc.at_css("li[id=#{fn_id(ref_num)}]") + backref_node = footnote_node.at_css("a[href=\"##{fnref_id(ref_num)}\"]") + + if ref_num =~ INTEGER_PATTERN && footnote_node && backref_node + link_node[:href] += rand_suffix + link_node[:id] += rand_suffix + footnote_node[:id] += rand_suffix + backref_node[:href] += rand_suffix + + # Sanitization stripped off class - add it back in + link_node.parent.append_class('footnote-ref') + backref_node.append_class('footnote-backref') + end + end + + doc + end + + private + + def random_number + @random_number ||= rand(10000) + end + + def fn_id(num) + "#{FOOTNOTE_ID_PREFIX}#{num}" + end + + def fnref_id(num) + "#{FOOTNOTE_LINK_ID_PREFIX}#{num}" + end + end + end +end diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb new file mode 100644 index 00000000000..a27d18facd1 --- /dev/null +++ b/lib/banzai/filter/front_matter_filter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class FrontMatterFilter < HTML::Pipeline::Filter + DELIM_LANG = { + '---' => 'yaml', + '+++' => 'toml', + ';;;' => 'json' + }.freeze + + DELIM = Regexp.union(DELIM_LANG.keys) + + PATTERN = %r{ + \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line + \s* + ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier) + \s* + ^(?<front_matter>.*?) # front matter (not greedy) + \s* + ^\k<delim> # closing front matter marker + \s* + }mx + + def call + html.sub(PATTERN) do |_match| + lang = $~[:lang].presence || DELIM_LANG[$~[:delim]] + + ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n") + end + end + end + end +end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 4bc82ecb4d6..0c1bbd2d250 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML Filter for parsing Gollum's tags in HTML. It's only parses the @@ -56,10 +58,12 @@ module Banzai # Pattern to match allowed image extensions ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze + # Do not perform linking inside these tags. + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + def call doc.search(".//text()").each do |node| - # Do not perform linking inside <code> blocks - next unless node.ancestors('code').empty? + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb index e008fd428b0..406c2d3c96b 100644 --- a/lib/banzai/filter/html_entity_filter.rb +++ b/lib/banzai/filter/html_entity_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'erb' module Banzai diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index 4cd9b02b76c..d8b9eb29cf5 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js module Banzai module Filter # HTML filter that moves the value of image `src` attributes to `data-src` @@ -5,7 +8,7 @@ module Banzai class ImageLazyLoadFilter < HTML::Pipeline::Filter def call doc.xpath('descendant-or-self::img').each do |img| - img['class'] ||= '' << 'lazy' + img.add_class('lazy') img['data-src'] = img['src'] img['src'] = LazyImageTagHelper.placeholder_image end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index f318c425962..01237303c27 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/image.js module Banzai module Filter # HTML filter that wraps links around inline images. diff --git a/lib/banzai/filter/inline_diff_filter.rb b/lib/banzai/filter/inline_diff_filter.rb index 73e82a4d7e3..5a1c0bee32d 100644 --- a/lib/banzai/filter/inline_diff_filter.rb +++ b/lib/banzai/filter/inline_diff_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/marks/inline_diff.js module Banzai module Filter class InlineDiffFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb index 7addf09be73..2963cba91e8 100644 --- a/lib/banzai/filter/issuable_reference_filter.rb +++ b/lib/banzai/filter/issuable_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter class IssuableReferenceFilter < AbstractReferenceFilter diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 1a415232545..8e2358694d4 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that appends state information to issuable links. @@ -16,7 +18,7 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| - next if !can_read_cross_project? && issuable.project != project + next if !can_read_cross_project? && cross_reference?(issuable) if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" @@ -29,7 +31,14 @@ module Banzai private def issuable_reference?(text, issuable) - text == issuable.reference_link_text(project || group) + CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) + end + + def cross_reference?(issuable) + return true if issuable.project != project + return true if issuable.respond_to?(:group) && issuable.group != group + + false end def can_read_cross_project? diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 6877cae8c55..f85be042999 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces issue references with links. References to diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a5f38046a43..f90a35952e5 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces label references with links. @@ -27,7 +29,7 @@ module Banzai if label yield match, label.id, project, namespace, $~ else - match + escape_html_entities(match) end end end @@ -46,7 +48,7 @@ module Banzai include_ancestor_groups: true, only_group_labels: true } else - { project_id: parent.id, + { project: parent, include_ancestor_groups: true } end @@ -100,6 +102,10 @@ module Banzai CGI.unescapeHTML(text.to_s) end + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + def object_link_title(object, matches) # use title of wrapped element instead nil diff --git a/lib/banzai/filter/markdown_engines/common_mark.rb b/lib/banzai/filter/markdown_engines/common_mark.rb index bc9597df894..d3af776db05 100644 --- a/lib/banzai/filter/markdown_engines/common_mark.rb +++ b/lib/banzai/filter/markdown_engines/common_mark.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # `CommonMark` markdown engine for GitLab's Banzai markdown filter. # This module is used in Banzai::Filter::MarkdownFilter. # Used gem is `commonmarker` which is a ruby wrapper for libcmark (CommonMark parser) @@ -18,7 +20,7 @@ module Banzai PARSE_OPTIONS = [ :FOOTNOTES, # parse footnotes. :STRIKETHROUGH_DOUBLE_TILDE, # parse strikethroughs by double tildes (as redcarpet does). - :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. + :VALIDATE_UTF8 # replace illegal sequences with the replacement character U+FFFD. ].freeze # The `:GITHUB_PRE_LANG` option is not used intentionally because @@ -30,8 +32,13 @@ module Banzai :DEFAULT # default rendering system. Nothing special. ].freeze - def initialize - @renderer = Banzai::Renderer::CommonMark::HTML.new(options: RENDER_OPTIONS) + RENDER_OPTIONS_SOURCEPOS = RENDER_OPTIONS + [ + :SOURCEPOS # enable embedding of source position information + ].freeze + + def initialize(context) + @context = context + @renderer = Banzai::Renderer::CommonMark::HTML.new(options: render_options) end def render(text) @@ -39,6 +46,12 @@ module Banzai @renderer.render(doc) end + + private + + def render_options + @context[:no_sourcepos] ? RENDER_OPTIONS : RENDER_OPTIONS_SOURCEPOS + end end end end diff --git a/lib/banzai/filter/markdown_engines/redcarpet.rb b/lib/banzai/filter/markdown_engines/redcarpet.rb index ac99941fefa..5b3f75096b1 100644 --- a/lib/banzai/filter/markdown_engines/redcarpet.rb +++ b/lib/banzai/filter/markdown_engines/redcarpet.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # `Redcarpet` markdown engine for GitLab's Banzai markdown filter. # This module is used in Banzai::Filter::MarkdownFilter. # Used gem is `redcarpet` which is a ruby library for markdown processing. @@ -18,7 +20,7 @@ module Banzai tables: true }.freeze - def initialize + def initialize(context = nil) html_renderer = Banzai::Renderer::Redcarpet::HTML.new @renderer = ::Redcarpet::Markdown.new(html_renderer, OPTIONS) end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index c1e2b680240..242e39f5495 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter def initialize(text, context = nil, result = nil) super(text, context, result) - @renderer = renderer(context[:markdown_engine]).new + @renderer = renderer(context[:markdown_engine]).new(context) @text = @text.delete("\r") end @@ -14,7 +16,7 @@ module Banzai private - DEFAULT_ENGINE = :redcarpet + DEFAULT_ENGINE = :common_mark def engine(engine_from_context) engine_from_context ||= DEFAULT_ENGINE diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index b6e784c886b..8dd5a8979c8 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -1,5 +1,10 @@ +# frozen_string_literal: true + require 'uri' +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/marks/math.js +# - app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 5cbdb01c130..7098767b583 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces merge request references with links. References @@ -25,7 +27,10 @@ module Banzai extras = super if commit_ref = object_link_commit_ref(object, matches) - return extras.unshift(commit_ref) + klass = reference_class(:commit, tooltip: false) + commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>) + + return extras.unshift(commit_ref_tag) end path = matches[:path] if matches.names.include?("path") diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index 65c131e08d9..f0adb83af8a 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index b144bd8cf54..fce042e8946 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces milestone references with links. class MilestoneReferenceFilter < AbstractReferenceFilter + include Gitlab::Utils::StrongMemoize + self.reference_type = :milestone def self.object_class @@ -11,16 +15,34 @@ module Banzai # Links to project milestones contain the IID, but when we're handling # 'regular' references, we need to use the global ID to disambiguate # between group and project milestones. - def find_object(project, id) - return unless project.is_a?(Project) + def find_object(parent, id) + return unless valid_context?(parent) + + find_milestone_with_finder(parent, id: id) + end + + def find_object_from_link(parent, iid) + return unless valid_context?(parent) - find_milestone_with_finder(project, id: id) + find_milestone_with_finder(parent, iid: iid) end - def find_object_from_link(project, iid) - return unless project.is_a?(Project) + def valid_context?(parent) + strong_memoize(:valid_context) do + group_context?(parent) || project_context?(parent) + end + end - find_milestone_with_finder(project, iid: iid) + def group_context?(parent) + strong_memoize(:group_context) do + parent.is_a?(Group) + end + end + + def project_context?(parent) + strong_memoize(:project_context) do + parent.is_a?(Project) + end end def references_in(text, pattern = Milestone.reference_pattern) @@ -42,13 +64,15 @@ module Banzai def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) project_path = full_project_path(namespace_ref, project_ref) - project = parent_from_ref(project_path) - return unless project && project.is_a?(Project) + # Returns group if project is not found by path + parent = parent_from_ref(project_path) + + return unless parent milestone_params = milestone_params(milestone_id, milestone_name) - find_milestone_with_finder(project, milestone_params) + find_milestone_with_finder(parent, milestone_params) end def milestone_params(iid, name) @@ -59,16 +83,28 @@ module Banzai end end - def find_milestone_with_finder(project, params) - finder_params = { project_ids: [project.id], order: nil, state: 'all' } + def find_milestone_with_finder(parent, params) + finder_params = milestone_finder_params(parent, params[:iid].present?) + + MilestonesFinder.new(finder_params).find_by(params) + end - # We don't support IID lookups for group milestones, because IIDs can - # clash between group and project milestones. - if project.group && !params[:iid] - finder_params[:group_ids] = [project.group.id] + def milestone_finder_params(parent, find_by_iid) + { order: nil, state: 'all' }.tap do |params| + params[:project_ids] = parent.id if project_context?(parent) + + # We don't support IID lookups because IIDs can clash between + # group/project milestones and group/subgroup milestones. + params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid end + end - MilestonesFinder.new(finder_params).find_by(params) + def self_and_ancestors_ids(parent) + if group_context?(parent) + parent.self_and_ancestors.select(:id) + elsif project_context?(parent) + parent.group&.self_and_ancestors&.select(:id) + end end def url_for_object(milestone, project) diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 28933c78966..caba8955bac 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "nokogiri" require "asciidoctor-plantuml/plantuml" diff --git a/lib/banzai/filter/project_reference_filter.rb b/lib/banzai/filter/project_reference_filter.rb new file mode 100644 index 00000000000..83cf45097ed --- /dev/null +++ b/lib/banzai/filter/project_reference_filter.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter that replaces project references with links. + class ProjectReferenceFilter < ReferenceFilter + self.reference_type = :project + + # Public: Find `namespace/project>` project references in text + # + # ProjectReferenceFilter.references_in(text) do |match, project| + # "<a href=...>#{project}></a>" + # end + # + # text - String text to search. + # + # Yields the String match, and the String project name. + # + # Returns a String replaced with the return of the block. + def self.references_in(text) + text.gsub(Project.markdown_reference_pattern) do |match| + yield match, "#{$~[:namespace]}/#{$~[:project]}" + end + end + + def call + ref_pattern = Project.markdown_reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + nodes.each do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + project_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_pattern_start + replace_link_node_with_href(node, link) do + project_link_filter(link, link_content: inner_html) + end + end + end + end + end + + doc + end + + # Replace `namespace/project>` project references in text with links to the referenced + # project page. + # + # text - String text to replace references in. + # link_content - Original content of the link being replaced. + # + # Returns a String with `namespace/project>` references replaced with links. All links + # have `gfm` and `gfm-project` class names attached for styling. + def project_link_filter(text, link_content: nil) + self.class.references_in(text) do |match, project_path| + cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do + if project = projects_hash[project_path.downcase] + link_to_project(project, link_content: link_content) || match + else + match + end + end + end + end + + # Returns a Hash containing all Project objects for the project + # references in the current document. + # + # The keys of this Hash are the project paths, the values the + # corresponding Project objects. + def projects_hash + @projects ||= Project.eager_load(:route, namespace: [:route]) + .where_full_path_in(projects) + .index_by(&:full_path) + .transform_keys(&:downcase) + end + + # Returns all projects referenced in the current document. + def projects + refs = Set.new + + nodes.each do |node| + node.to_html.scan(Project.markdown_reference_pattern) do + refs << "#{$~[:namespace]}/#{$~[:project]}" + end + end + + refs.to_a + end + + private + + def urls + Gitlab::Routing.url_helpers + end + + def link_class + reference_class(:project) + end + + def link_to_project(project, link_content: nil) + url = urls.project_url(project, only_path: context[:only_path]) + data = data_attribute(project: project.id) + content = link_content || project.to_reference_with_postfix + + link_tag(url, data, content, project.name) + end + + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) + end + end + end +end diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index caf11fe94c4..1f091f594f8 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that removes references to records that the current user does diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2f023f4f242..42f9b3a689c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js module Banzai module Filter # Base class for GitLab Flavored Markdown reference filters. @@ -65,8 +68,12 @@ module Banzai context[:skip_project_check] end - def reference_class(type) - "gfm gfm-#{type} has-tooltip" + def reference_class(type, tooltip: true) + gfm_klass = "gfm gfm-#{type}" + + return gfm_klass unless tooltip + + "#{gfm_klass} has-tooltip" end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 262458a872a..93e6d6470f1 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'uri' module Banzai @@ -56,9 +58,15 @@ module Banzai path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') elsif project path_parts.unshift(relative_url_root, project.full_path) + else + path_parts.unshift(relative_url_root) end - path = Addressable::URI.escape(File.join(*path_parts)) + begin + path = Addressable::URI.escape(File.join(*path_parts)) + rescue Addressable::URI::InvalidURIError + return + end html_attr.value = if context[:only_path] diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6786b9d07b6..a4a06eae7b7 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,34 +1,30 @@ +# frozen_string_literal: true + module Banzai module Filter # Sanitize HTML # # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ - - def whitelist - whitelist = super + include Gitlab::Utils::StrongMemoize - customize_whitelist(whitelist) + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze + TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze - whitelist + def whitelist + strong_memoize(:whitelist) do + customize_whitelist(super.deep_dup) + end end private - def customized?(transformers) - transformers.last.source_location[0] == __FILE__ - end - def customize_whitelist(whitelist) - # Only push these customizations once - return if customized?(whitelist[:transformers]) - - # Allow table alignment; we whitelist specific style properties in a + # Allow table alignment; we whitelist specific text-align values in a # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) + whitelist[:css] = { properties: ['text-align'] } # Allow span elements whitelist[:elements].push('span') @@ -45,14 +41,16 @@ module Banzai whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) + # Allow the 'data-sourcepos' from CommonMark on all elements + whitelist[:attributes][:all].push('data-sourcepos') + # Disallow `name` attribute globally, allow on `a` whitelist[:attributes][:all].delete('name') whitelist[:attributes]['a'].push('name') - # Allow any protocol in `a` elements... + # Allow any protocol in `a` elements + # and then remove links with unsafe protocols whitelist[:protocols].delete('a') - - # ...but then remove links with unsafe protocols whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements @@ -61,6 +59,12 @@ module Banzai # Remove any `style` properties not required for table alignment whitelist[:transformers].push(self.class.remove_unsafe_table_style) + # Allow `id` in a and li elements for footnotes + # and remove any `id` properties not matching for footnotes + whitelist[:attributes]['a'].push('id') + whitelist[:attributes]['li'] = %w(id) + whitelist[:transformers].push(self.class.remove_non_footnote_ids) + whitelist end @@ -116,6 +120,20 @@ module Banzai end end end + + def remove_non_footnote_ids + lambda do |env| + node = env[:node] + + return unless node.name == 'a' || node.name == 'li' + return unless node.has_attribute?('id') + + return if node.name == 'a' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LINK_REFERENCE_PATTERN + return if node.name == 'li' && node['id'] =~ Banzai::Filter::FootnoteFilter::FOOTNOTE_LI_REFERENCE_PATTERN + + node.remove_attribute('id') + end + end end end end diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb index c2976aeb7c6..45b259a2faf 100644 --- a/lib/banzai/filter/set_direction_filter.rb +++ b/lib/banzai/filter/set_direction_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that sets dir="auto" for RTL languages support diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 881e10afb9f..f4b6edb6174 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces snippet references with links. References to diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb new file mode 100644 index 00000000000..00dbf2d3130 --- /dev/null +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'uri' + +module Banzai + module Filter + # HTML Filter for markdown links with spaces in the URLs + # + # Based on Banzai::Filter::AutolinkFilter + # + # CommonMark does not allow spaces in the url portion of a link/url. + # For example, `[example](page slug)` is not valid. + # Neither is `![example](test image.jpg)`. However, particularly + # in our wikis, we support (via RedCarpet) this type of link, allowing + # wiki pages to be easily linked by their title. This filter adds that functionality. + # + # This is a small extension to the CommonMark spec. If they start allowing + # spaces in urls, we could then remove this filter. + # + # Note: Filter::SanitizationFilter should always be run sometime after this filter + # to prevent XSS attacks + # + class SpacedLinkFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match a standard markdown link + # + # Rubular: http://rubular.com/r/2EXEQ49rg5 + LINK_OR_IMAGE_PATTERN = %r{ + (?<preview_operator>!)? + \[(?<text>.+?)\] + \( + (?<new_link>.+?) + (?<title>\ ".+?")? + \) + }x + + # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked + IGNORE_PARENTS = %w(a code kbd pre script style).to_set + + # The XPath query to use for finding text nodes to parse. + TEXT_QUERY = %Q(descendant-or-self::text()[ + not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) + and contains(., ']\(') + ]).freeze + + def call + return doc if context[:markdown_engine] == :redcarpet + + doc.xpath(TEXT_QUERY).each do |node| + content = node.to_html + + next unless content.match(LINK_OR_IMAGE_PATTERN) + + html = spaced_link_filter(content) + + next if html == content + + node.replace(html) + end + + doc + end + + private + + def spaced_link_match(link) + match = LINK_OR_IMAGE_PATTERN.match(link) + return link unless match + + # escape the spaces in the url so that it's a valid markdown link, + # then run it through the markdown processor again, let it do its magic + html = Banzai::Filter::MarkdownFilter.call(transform_markdown(match), context) + + # link is wrapped in a <p>, so strip that off + p_node = Nokogiri::HTML.fragment(html).at_css('p') + p_node ? p_node.children.to_html : html + end + + def spaced_link_filter(text) + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:| + spaced_link_match(link) + end + end + + def transform_markdown(match) + preview_operator, text, new_link, title = process_match(match) + + "#{preview_operator}[#{text}](#{new_link}#{title})" + end + + def process_match(match) + [ + match[:preview_operator], + match[:text], + match[:new_link].gsub(' ', '%20'), + match[:title] + ] + end + end + end +end diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb new file mode 100644 index 00000000000..9950db373d8 --- /dev/null +++ b/lib/banzai/filter/suggestion_filter.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js +module Banzai + module Filter + class SuggestionFilter < HTML::Pipeline::Filter + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-suggestion'.freeze + + def call + return doc unless suggestions_filter_enabled? + + doc.search('pre.suggestion > code').each do |node| + node.add_class(TAG_CLASS) + end + + doc + end + + def suggestions_filter_enabled? + context[:suggestions_filter_enabled] + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 6dbf0d68fe8..bcf77861f10 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,6 +1,9 @@ +# frozen_string_literal: true + require 'rouge/plugins/common_mark' require 'rouge/plugins/redcarpet' +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter # HTML Filter to highlight fenced code blocks @@ -15,7 +18,7 @@ module Banzai end def highlight_node(node) - css_classes = 'code highlight js-syntax-highlight' + css_classes = +'code highlight js-syntax-highlight' lang = node.attr('lang') retried = false @@ -67,7 +70,7 @@ module Banzai end def use_rouge?(language) - %w(math mermaid plantuml).exclude?(language) + %w(math mermaid plantuml suggestion).exclude?(language) end end end diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 97244159985..f2ae17b44fa 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/table_of_contents.js module Banzai module Filter # HTML filter that adds an anchor child element to all Headers in a @@ -19,7 +22,7 @@ module Banzai def call return doc if context[:no_header_anchors] - result[:toc] = "" + result[:toc] = +"" headers = Hash.new(0) header_root = current_header = HeaderNode.new @@ -92,7 +95,7 @@ module Banzai def text return '' unless node - @text ||= node.text + @text ||= EscapeUtils.escape_html(node.text) end private diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 9fa5f589f3e..c6b402575cb 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,5 +1,11 @@ +# frozen_string_literal: true + require 'task_list/filter' +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/nodes/ordered_task_list.js +# - app/assets/javascripts/behaviors/markdown/nodes/task_list.js +# - app/assets/javascripts/behaviors/markdown/nodes/task_list_item.js module Banzai module Filter class TaskListFilter < TaskList::Filter diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index c7fa8a8119f..8cda67867a8 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter # HTML filter that replaces user or group references with links. @@ -104,7 +106,7 @@ module Banzai end def link_class - reference_class(:project_member) + reference_class(:project_member, tooltip: false) end def link_to_all(link_content: nil) diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index 35cb10eae5d..0fff104cf91 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -1,3 +1,6 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/video.js module Banzai module Filter # Find every image that isn't already wrapped in an `a` tag, and that has diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 269d5bf74fa..1728a442533 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -1,4 +1,4 @@ -require 'uri' +# frozen_string_literal: true module Banzai module Filter @@ -11,8 +11,12 @@ module Banzai def call return doc unless project_wiki? - doc.search('a:not(.gfm)').each do |el| - process_link_attr el.attribute('href') + doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) } + doc.search('video').each { |el| process_link_attr(el.attribute('src')) } + doc.search('img').each do |el| + attr = el.attribute('data-src') || el.attribute('src') + + process_link_attr(attr) end doc diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 072d24e5a11..f4cc8beeb52 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Filter class WikiLinkFilter < HTML::Pipeline::Filter @@ -10,11 +12,16 @@ module Banzai def apply_rules # Special case: relative URLs beginning with `/uploads/` refer to - # user-uploaded files and will be handled elsewhere. - return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/') + # user-uploaded files will be handled elsewhere. + return @uri.to_s if public_upload? + + # Special case: relative URLs beginning with Wikis::CreateAttachmentService::ATTACHMENT_PATH + # refer to user-uploaded files to the wiki repository. + unless repository_upload? + apply_file_link_rules! + apply_hierarchical_link_rules! + end - apply_file_link_rules! - apply_hierarchical_link_rules! apply_relative_link_rules! @uri.to_s end @@ -39,6 +46,14 @@ module Banzai @uri = Addressable::URI.parse(link) end end + + def public_upload? + @uri.relative? && @uri.path.starts_with?('/uploads/') + end + + def repository_upload? + @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH) + end end end end diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb deleted file mode 100644 index 58e3e81209e..00000000000 --- a/lib/banzai/filter/yaml_front_matter_filter.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Banzai - module Filter - class YamlFrontMatterFilter < HTML::Pipeline::Filter - DELIM = '---'.freeze - - # Hat-tip to Middleman: https://git.io/v2e0z - PATTERN = %r{ - \A(?:[^\r\n]*coding:[^\r\n]*\r?\n)? - (?<start>#{DELIM})[ ]*\r?\n - (?<frontmatter>.*?)[ ]*\r?\n? - ^(?<stop>#{DELIM})[ ]*\r?\n? - \r?\n? - (?<content>.*) - }mx.freeze - - def call - match = PATTERN.match(html) - - return html unless match - - "```yaml\n#{match['frontmatter']}\n```\n\n#{match['content']}" - end - end - end -end diff --git a/lib/banzai/filter_array.rb b/lib/banzai/filter_array.rb index 77835a14027..818af4643a7 100644 --- a/lib/banzai/filter_array.rb +++ b/lib/banzai/filter_array.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai class FilterArray < Array # Insert a value immediately after another value diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index ae7dc71e7eb..341dbb74fe0 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai # Extract references to issuables from multiple documents @@ -7,13 +9,11 @@ module Banzai # so we can avoid N+1 queries problem class IssuableExtractor - QUERY = %q( - descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] - [@data-reference-type="issue" or @data-reference-type="merge_request"] - ).freeze - attr_reader :context + ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'.freeze + MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"'.freeze + # context - An instance of Banzai::RenderContext. def initialize(context) @context = context @@ -22,21 +22,38 @@ module Banzai # Returns Hash in the form { node => issuable_instance } def extract(documents) nodes = documents.flat_map do |document| - document.xpath(QUERY) + document.xpath(query) end - issue_parser = Banzai::ReferenceParser::IssueParser.new(context) + # The project or group for the issuable might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes(nodes).select { |node, issuable| issuable.project || issuable.group } + end + + private - merge_request_parser = + def issuables_for_nodes(nodes) + parsers.each_with_object({}) do |parser, result| + result.merge!(parser.records_for_nodes(nodes)) + end + end + + def parsers + [ + Banzai::ReferenceParser::IssueParser.new(context), Banzai::ReferenceParser::MergeRequestParser.new(context) + ] + end - issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( - merge_request_parser.records_for_nodes(nodes) + def query + %Q( + descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] + [#{reference_types.join(' or ')}] ) + end - # The project for the issue/MR might be pending for deletion! - # Filter them out because we don't care about them. - issuables_for_nodes.select { |node, issuable| issuable.project } + def reference_types + [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE] end end end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index a176f1e261b..75661ffa233 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai # Class for rendering multiple objects (e.g. Note instances) in a single pass, # using +render_field+ to benefit from caching in the database. Rendering and @@ -38,6 +40,7 @@ module Banzai redacted_data = redacted[index] object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html(save_options).html_safe) # rubocop:disable GitlabSecurity/PublicSend object.user_visible_reference_count = redacted_data[:visible_reference_count] if object.respond_to?(:user_visible_reference_count) + object.total_reference_count = redacted_data[:total_reference_count] if object.respond_to?(:total_reference_count) end end diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb index 142a9962eb1..e8a81bebaa9 100644 --- a/lib/banzai/pipeline.rb +++ b/lib/banzai/pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline def self.[](name) diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb index 1048b927cd3..cc4af280872 100644 --- a/lib/banzai/pipeline/ascii_doc_pipeline.rb +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class AsciiDocPipeline < BasePipeline diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 9694e4bc23f..c632910585d 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -1,10 +1,13 @@ +# frozen_string_literal: true + module Banzai module Pipeline class AtomPipeline < FullPipeline def self.transform_context(context) super(context).merge( only_path: false, - xhtml: true + xhtml: true, + no_sourcepos: true ) end end diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index 3ae3bed570d..87d1cf9912f 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class BasePipeline diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb index 5dd572de3a1..580b5b72474 100644 --- a/lib/banzai/pipeline/broadcast_message_pipeline.rb +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class BroadcastMessagePipeline < DescriptionPipeline @@ -12,6 +14,12 @@ module Banzai Filter::ExternalLinkFilter ] end + + def self.transform_context(context) + super(context).merge( + no_sourcepos: true + ) + end end end end diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb index 60190f8d9dd..56b424dc8e0 100644 --- a/lib/banzai/pipeline/combined_pipeline.rb +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline module CombinedPipeline diff --git a/lib/banzai/pipeline/commit_description_pipeline.rb b/lib/banzai/pipeline/commit_description_pipeline.rb index 607c2731ed3..e8ec7453f0f 100644 --- a/lib/banzai/pipeline/commit_description_pipeline.rb +++ b/lib/banzai/pipeline/commit_description_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class CommitDescriptionPipeline < SingleLinePipeline diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 042fb2e6e14..d5ff9b025cc 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class DescriptionPipeline < FullPipeline diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 8f5f144d582..13e6a990407 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class EmailPipeline < FullPipeline @@ -9,7 +11,9 @@ module Banzai def self.transform_context(context) super(context).merge( - only_path: false + only_path: false, + emailable_links: true, + no_sourcepos: true ) end end diff --git a/lib/banzai/pipeline/emoji_pipeline.rb b/lib/banzai/pipeline/emoji_pipeline.rb new file mode 100644 index 00000000000..a1b522f4303 --- /dev/null +++ b/lib/banzai/pipeline/emoji_pipeline.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class EmojiPipeline < BasePipeline + # These filters will only perform sanitization of the content, preventing + # XSS, and replace emoji. + def self.filters + @filters ||= FilterArray[ + Filter::HtmlEntityFilter, + Filter::SanitizationFilter, + Filter::EmojiFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index 3c974f73176..a5b1cbdd030 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index a1f24e8b093..30cafd11834 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,15 +1,21 @@ +# frozen_string_literal: true + module Banzai module Pipeline class GfmPipeline < BasePipeline - # These filters convert GitLab Flavored Markdown (GFM) to HTML. - # The handlers defined in app/assets/javascripts/behaviors/markdown/copy_as_gfm.js - # consequently convert that same HTML to GFM to be copied to the clipboard. - # Every filter that generates HTML from GFM should have a handler in - # app/assets/javascripts/behaviors/markdown/copy_as_gfm.js, in reverse order. + # These filters transform GitLab Flavored Markdown (GFM) to HTML. + # The nodes and marks referenced in app/assets/javascripts/behaviors/markdown/editor_extensions.js + # consequently transform that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a node or mark in + # app/assets/javascripts/behaviors/markdown/editor_extensions.js. # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ Filter::PlantumlFilter, + + # Must always be before the SanitizationFilter to prevent XSS attacks + Filter::SpacedLinkFilter, + Filter::SanitizationFilter, Filter::SyntaxHighlightFilter, @@ -23,8 +29,22 @@ module Banzai Filter::TableOfContentsFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter, + Filter::SuggestionFilter, + Filter::FootnoteFilter, + + *reference_filters, + + Filter::TaskListFilter, + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter + ] + end + def self.reference_filters + [ Filter::UserReferenceFilter, + Filter::ProjectReferenceFilter, Filter::IssueReferenceFilter, Filter::ExternalIssueReferenceFilter, Filter::MergeRequestReferenceFilter, @@ -32,23 +52,14 @@ module Banzai Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, - Filter::MilestoneReferenceFilter, - - Filter::TaskListFilter, - Filter::InlineDiffFilter, - - Filter::SetDirectionFilter + Filter::MilestoneReferenceFilter ] end def self.transform_context(context) context[:only_path] = true unless context.key?(:only_path) - context.merge( - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url - ) + context end end end diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb new file mode 100644 index 00000000000..725cccc4b2b --- /dev/null +++ b/lib/banzai/pipeline/label_pipeline.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class LabelPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::LabelReferenceFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index c56d908009f..db79a22549c 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class MarkupPipeline < BasePipeline diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb index 7890f20f716..4480d7ede05 100644 --- a/lib/banzai/pipeline/note_pipeline.rb +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class NotePipeline < FullPipeline diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 3f45db21869..b64f13cde47 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index dcd52bc03c7..7eaad6d7560 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -1,12 +1,21 @@ +# frozen_string_literal: true + module Banzai module Pipeline class PostProcessPipeline < BasePipeline def self.filters - FilterArray[ + @filters ||= FilterArray[ + *internal_link_filters, + Filter::AbsoluteLinkFilter + ] + end + + def self.internal_link_filters + [ Filter::RedactorFilter, Filter::RelativeLinkFilter, Filter::IssuableStateFilter, - Filter::AbsoluteLinkFilter + Filter::SuggestionFilter ] end diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb index 6cf219661d3..4c2b4ca1665 100644 --- a/lib/banzai/pipeline/pre_process_pipeline.rb +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Banzai module Pipeline class PreProcessPipeline < BasePipeline def self.filters FilterArray[ - Filter::YamlFrontMatterFilter, + Filter::FrontMatterFilter, Filter::BlockquoteFenceFilter, ] end diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb index 270990e7ab4..88651892acc 100644 --- a/lib/banzai/pipeline/relative_link_pipeline.rb +++ b/lib/banzai/pipeline/relative_link_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class RelativeLinkPipeline < BasePipeline diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 1929099931b..72374207a8f 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class SingleLinePipeline < GfmPipeline @@ -10,15 +12,27 @@ module Banzai Filter::AutolinkFilter, Filter::ExternalLinkFilter, + *reference_filters + ] + end + + def self.reference_filters + [ Filter::UserReferenceFilter, Filter::IssueReferenceFilter, Filter::ExternalIssueReferenceFilter, Filter::MergeRequestReferenceFilter, Filter::SnippetReferenceFilter, Filter::CommitRangeReferenceFilter, - Filter::CommitReferenceFilter, + Filter::CommitReferenceFilter ] end + + def self.transform_context(context) + super(context).merge( + no_sourcepos: true + ) + end end end end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index c37b8e71cb0..97a03895ff3 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Pipeline class WikiPipeline < FullPipeline diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index a19a05e8c0d..55aa5fa66c3 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Querying module_function diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 28928d6f376..7db5f5e1f7d 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai # Class for removing Markdown references a certain user is not allowed to # view. @@ -37,7 +39,13 @@ module Banzai all_document_nodes.each do |entry| nodes_for_document = entry[:nodes] - doc_data = { document: entry[:document], visible_reference_count: nodes_for_document.count } + + doc_data = { + document: entry[:document], + total_reference_count: nodes_for_document.count, + visible_reference_count: nodes_for_document.count + } + metadata << doc_data nodes_for_document.each do |node| diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 78588299c18..3fc3ae02088 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb index 557bec4316e..efe15096f08 100644 --- a/lib/banzai/reference_parser.rb +++ b/lib/banzai/reference_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser # Returns the reference parser class for the given type diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 68752f5bb5a..8419769085a 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser # Base class for reference parsing classes. @@ -166,7 +168,7 @@ module Banzai # objects that have not yet been queried. For objects that have already # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] to_query = ids - cache.keys @@ -215,7 +217,7 @@ module Banzai # def projects_for_nodes(nodes) @projects_for_nodes ||= - grouped_objects_for_nodes(nodes, Project, 'data-project') + grouped_objects_for_nodes(nodes, Project.includes(:project_feature), 'data-project') end def can?(user, permission, subject = :global) @@ -248,7 +250,7 @@ module Banzai end def collection_cache - RequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| + Gitlab::SafeRequestStore[:banzai_collection_cache] ||= Hash.new do |hash, key| hash[key] = {} end end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 30dc87248b4..0bfb6a92020 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class CommitParser < BaseParser diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 2920e886938..480eefd5c4d 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class CommitRangeParser < BaseParser diff --git a/lib/banzai/reference_parser/directly_addressed_user_parser.rb b/lib/banzai/reference_parser/directly_addressed_user_parser.rb index 77df9bbd024..1f18f82b916 100644 --- a/lib/banzai/reference_parser/directly_addressed_user_parser.rb +++ b/lib/banzai/reference_parser/directly_addressed_user_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class DirectlyAddressedUserParser < UserParser diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb index 08b8a4c9a0f..7f366f0f8ab 100644 --- a/lib/banzai/reference_parser/epic_parser.rb +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser # The actual parser is implemented in the EE mixin diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index 1802cd04854..029b09dcd25 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class ExternalIssueParser < BaseParser diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb index fad127d7e5b..f8c26288017 100644 --- a/lib/banzai/reference_parser/issuable_parser.rb +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class IssuableParser < BaseParser diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index 7b5915899cf..97c7173ac0f 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class IssueParser < IssuableParser diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb index 30e2a012f09..398cc45fea0 100644 --- a/lib/banzai/reference_parser/label_parser.rb +++ b/lib/banzai/reference_parser/label_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class LabelParser < BaseParser diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index a370ff5b5b3..e8147ac591a 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class MergeRequestParser < IssuableParser @@ -14,11 +16,12 @@ module Banzai # Eager loading these ensures we don't end up running dozens of # queries in this process. target_project: [ - { namespace: :owner }, + { namespace: [:owner, :route] }, { group: [:owners, :group_members] }, :invited_groups, :project_members, - :project_feature + :project_feature, + :route ] }), self.class.data_attribute diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb index 68675abe22a..925d736fb9a 100644 --- a/lib/banzai/reference_parser/milestone_parser.rb +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class MilestoneParser < BaseParser diff --git a/lib/banzai/reference_parser/project_parser.rb b/lib/banzai/reference_parser/project_parser.rb new file mode 100644 index 00000000000..b4e3a55b4f1 --- /dev/null +++ b/lib/banzai/reference_parser/project_parser.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Banzai + module ReferenceParser + class ProjectParser < BaseParser + include Gitlab::Utils::StrongMemoize + + self.reference_type = :project + + def references_relation + Project + end + + private + + # Returns an Array of Project ids that can be read by the given user. + # + # user - The User for which to check the projects + def readable_project_ids_for(user) + @project_ids_by_user ||= {} + @project_ids_by_user[user] ||= + Project.public_or_visible_to_user(user).where("projects.id IN (?)", @projects_for_nodes.values.map(&:id)).pluck(:id) + end + + def can_read_reference?(user, ref_project, node) + readable_project_ids_for(user).include?(ref_project.try(:id)) + end + end + end +end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb index 3ade168b566..6f6ac08de04 100644 --- a/lib/banzai/reference_parser/snippet_parser.rb +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class SnippetParser < BaseParser diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index ceb7f1d165c..067b06b7590 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module ReferenceParser class UserParser < BaseParser diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 0050295eeda..c7239a5eaa6 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Renderer # Convert a Markdown String into an HTML-safe String of HTML diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb index 46b609c36b0..837665451a1 100644 --- a/lib/banzai/renderer/common_mark/html.rb +++ b/lib/banzai/renderer/common_mark/html.rb @@ -1,18 +1,16 @@ +# frozen_string_literal: true + module Banzai module Renderer module CommonMark class HTML < CommonMarker::HtmlRenderer def code_block(node) block do - code = node.string_content - lang = node.fence_info - lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' - result = - "<pre>" \ - "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ - "</pre>" - - out(result) + out("<pre#{sourcepos(node)}><code") + out(' lang="', node.fence_info, '"') if node.fence_info.present? + out('>') + out(escape_html(node.string_content)) + out('</code></pre>') end end end diff --git a/lib/banzai/renderer/redcarpet/html.rb b/lib/banzai/renderer/redcarpet/html.rb index 30e815f1224..84931fdc784 100644 --- a/lib/banzai/renderer/redcarpet/html.rb +++ b/lib/banzai/renderer/redcarpet/html.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Banzai module Renderer module Redcarpet diff --git a/lib/banzai/request_store_reference_cache.rb b/lib/banzai/request_store_reference_cache.rb index 426131442a2..91fb489b72d 100644 --- a/lib/banzai/request_store_reference_cache.rb +++ b/lib/banzai/request_store_reference_cache.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module Banzai module RequestStoreReferenceCache def cached_call(request_store_key, cache_key, path: []) - if RequestStore.active? - cache = RequestStore[request_store_key] ||= Hash.new do |hash, key| + if Gitlab::SafeRequestStore.active? + cache = Gitlab::SafeRequestStore[request_store_key] ||= Hash.new do |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } end diff --git a/lib/banzai/suggestions_parser.rb b/lib/banzai/suggestions_parser.rb new file mode 100644 index 00000000000..09f36635020 --- /dev/null +++ b/lib/banzai/suggestions_parser.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module SuggestionsParser + # Returns the content of each suggestion code block. + # + def self.parse(text) + html = Banzai.render(text, project: nil, no_original_data: true) + doc = Nokogiri::HTML(html) + + doc.search('pre.suggestion').map { |node| node.text } + end + end +end diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb index f8ee7e0f9ae..1343f424c51 100644 --- a/lib/bitbucket/client.rb +++ b/lib/bitbucket/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket class Client attr_reader :connection diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb index a78495dbf5e..9c496daccaa 100644 --- a/lib/bitbucket/collection.rb +++ b/lib/bitbucket/collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket class Collection < Enumerator def initialize(paginator) diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index ba5a9e2f04c..0041634f9e3 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket class Connection DEFAULT_API_VERSION = '2.0'.freeze diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb index efe10542f19..3cde11babee 100644 --- a/lib/bitbucket/error/unauthorized.rb +++ b/lib/bitbucket/error/unauthorized.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Error Unauthorized = Class.new(StandardError) diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb index 2b0a3fe7b1a..7cc1342ad65 100644 --- a/lib/bitbucket/page.rb +++ b/lib/bitbucket/page.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket class Page attr_reader :attrs, :items diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb index 135d0d55674..0d004592e67 100644 --- a/lib/bitbucket/paginator.rb +++ b/lib/bitbucket/paginator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket class Paginator PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb index 800d5a075c6..bb8dcd91ad5 100644 --- a/lib/bitbucket/representation/base.rb +++ b/lib/bitbucket/representation/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class Base diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb index 4937aa9728f..1b8dc27793a 100644 --- a/lib/bitbucket/representation/comment.rb +++ b/lib/bitbucket/representation/comment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class Comment < Representation::Base diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 44bcbc250b3..a88797cdab9 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class Issue < Representation::Base diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index eebf8093380..6a0e8b354bf 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class PullRequest < Representation::Base diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index c52acbc3ddc..34dbf9ad22d 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class PullRequestComment < Comment diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index 59b0fda8e14..c5bfc91e43d 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class Repo < Representation::Base diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb index ba6b7667b49..2b45d751e70 100644 --- a/lib/bitbucket/representation/user.rb +++ b/lib/bitbucket/representation/user.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Bitbucket module Representation class User < Representation::Base diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb new file mode 100644 index 00000000000..6a608058813 --- /dev/null +++ b/lib/bitbucket_server/client.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module BitbucketServer + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def pull_requests(project_key, repo) + path = "/projects/#{project_key}/repos/#{repo}/pull-requests?state=ALL" + get_collection(path, :pull_request) + end + + def activities(project_key, repo, pull_request_id) + path = "/projects/#{project_key}/repos/#{repo}/pull-requests/#{pull_request_id}/activities" + get_collection(path, :activity) + end + + def repo(project, repo_name) + parsed_response = connection.get("/projects/#{project}/repos/#{repo_name}") + BitbucketServer::Representation::Repo.new(parsed_response) + end + + def repos(page_offset: 0, limit: nil) + path = "/repos" + get_collection(path, :repo, page_offset: page_offset, limit: limit) + end + + def create_branch(project_key, repo, branch_name, sha) + payload = { + name: branch_name, + startPoint: sha, + message: 'GitLab temporary branch for import' + } + + connection.post("/projects/#{project_key}/repos/#{repo}/branches", payload.to_json) + end + + def delete_branch(project_key, repo, branch_name, sha) + payload = { + name: Gitlab::Git::BRANCH_REF_PREFIX + branch_name, + dryRun: false + } + + connection.delete(:branches, "/projects/#{project_key}/repos/#{repo}/branches", payload.to_json) + end + + private + + def get_collection(path, type, page_offset: 0, limit: nil) + paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit) + BitbucketServer::Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb new file mode 100644 index 00000000000..7e4b2277bbe --- /dev/null +++ b/lib/bitbucket_server/collection.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module BitbucketServer + class Collection < Enumerator + attr_reader :paginator + + delegate :page_offset, :has_next_page?, to: :paginator + + def initialize(paginator) + @paginator = paginator + + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def current_page + return 1 if page_offset <= 1 + + [1, page_offset].max + end + + def prev_page + return nil unless current_page > 1 + + current_page - 1 + end + + def next_page + return nil unless has_next_page? + + current_page + 1 + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb new file mode 100644 index 00000000000..9c14b26c65a --- /dev/null +++ b/lib/bitbucket_server/connection.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module BitbucketServer + class Connection + include ActionView::Helpers::SanitizeHelper + + DEFAULT_API_VERSION = '1.0' + SEPARATOR = '/' + + NETWORK_ERRORS = [ + SocketError, + OpenSSL::SSL::SSLError, + Errno::ECONNRESET, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Net::OpenTimeout, + Net::ReadTimeout, + Gitlab::HTTP::BlockedUrlError + ].freeze + + attr_reader :api_version, :base_uri, :username, :token + + ConnectionError = Class.new(StandardError) + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options[:base_uri] + @username = options[:user] + @token = options[:password] + end + + def get(path, extra_query = {}) + response = Gitlab::HTTP.get(build_url(path), + basic_auth: auth, + headers: accept_headers, + query: extra_query) + + check_errors!(response) + + response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e + end + + def post(path, body) + response = Gitlab::HTTP.post(build_url(path), + basic_auth: auth, + headers: post_headers, + body: body) + + check_errors!(response) + + response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e + end + + # We need to support two different APIs for deletion: + # + # /rest/api/1.0/projects/{projectKey}/repos/{repositorySlug}/branches/default + # /rest/branch-utils/1.0/projects/{projectKey}/repos/{repositorySlug}/branches + def delete(resource, path, body) + url = delete_url(resource, path) + + response = Gitlab::HTTP.delete(url, + basic_auth: auth, + headers: post_headers, + body: body) + + check_errors!(response) + + response.parsed_response + rescue *NETWORK_ERRORS => e + raise ConnectionError, e + end + + private + + def check_errors!(response) + raise ConnectionError, "Response is not valid JSON" unless response.parsed_response.is_a?(Hash) + + return if response.code >= 200 && response.code < 300 + + details = sanitize(response.parsed_response.dig('errors', 0, 'message')) + message = "Error #{response.code}" + message += ": #{details}" if details + + raise ConnectionError, message + rescue JSON::ParserError + raise ConnectionError, "Unable to parse the server response as JSON" + end + + def auth + @auth ||= { username: username, password: token } + end + + def accept_headers + @accept_headers ||= { 'Accept' => 'application/json' } + end + + def post_headers + @post_headers ||= accept_headers.merge({ 'Content-Type' => 'application/json' }) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + Gitlab::Utils.append_path(root_url, path) + end + + def root_url + Gitlab::Utils.append_path(base_uri, "rest/api/#{api_version}") + end + + def delete_url(resource, path) + if resource == :branches + Gitlab::Utils.append_path(base_uri, "rest/branch-utils/#{api_version}#{path}") + else + build_url(path) + end + end + end +end diff --git a/lib/bitbucket_server/page.rb b/lib/bitbucket_server/page.rb new file mode 100644 index 00000000000..5d9a3168876 --- /dev/null +++ b/lib/bitbucket_server/page.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module BitbucketServer + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + !attrs.fetch(:isLastPage, true) + end + + def next + attrs.fetch(:nextPageStart) + end + + private + + def parse_attrs(raw) + raw.slice('size', 'nextPageStart', 'isLastPage').symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + BitbucketServer::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb new file mode 100644 index 00000000000..9eda1c921b2 --- /dev/null +++ b/lib/bitbucket_server/paginator.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module BitbucketServer + class Paginator + PAGE_LENGTH = 25 + + attr_reader :page_offset + + def initialize(connection, url, type, page_offset: 0, limit: nil) + @connection = connection + @type = type + @url = url + @page = nil + @page_offset = page_offset + @limit = limit + @total = 0 + end + + def items + raise StopIteration unless has_next_page? + raise StopIteration if over_limit? + + @page = fetch_next_page + @total += @page.items.count + @page.items + end + + def has_next_page? + page.nil? || page.next? + end + + private + + attr_reader :connection, :page, :url, :type, :limit + + def over_limit? + return false unless @limit + + @limit.positive? && @total >= @limit + end + + def next_offset + page.nil? ? starting_offset : page.next + end + + def starting_offset + [0, page_offset - 1].max * max_per_page + end + + def max_per_page + limit || PAGE_LENGTH + end + + def fetch_next_page + parsed_response = connection.get(@url, start: next_offset, limit: max_per_page) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket_server/representation/activity.rb b/lib/bitbucket_server/representation/activity.rb new file mode 100644 index 00000000000..08bf30a5d1e --- /dev/null +++ b/lib/bitbucket_server/representation/activity.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + class Activity < Representation::Base + def comment? + action == 'COMMENTED' + end + + def inline_comment? + !!(comment? && comment_anchor) + end + + def comment + return unless comment? + + @comment ||= + if inline_comment? + PullRequestComment.new(raw) + else + Comment.new(raw) + end + end + + # TODO Move this into MergeEvent + def merge_event? + action == 'MERGED' + end + + def committer_user + commit.dig('committer', 'displayName') + end + + def committer_email + commit.dig('committer', 'emailAddress') + end + + def merge_timestamp + timestamp = commit['committerTimestamp'] + + self.class.convert_timestamp(timestamp) + end + + def merge_commit + commit['id'] + end + + def created_at + self.class.convert_timestamp(created_date) + end + + private + + def commit + raw.fetch('commit', {}) + end + + def action + raw['action'] + end + + def comment_anchor + raw['commentAnchor'] + end + + def created_date + raw['createdDate'] + end + end + end +end diff --git a/lib/bitbucket_server/representation/base.rb b/lib/bitbucket_server/representation/base.rb new file mode 100644 index 00000000000..a1961bae6ef --- /dev/null +++ b/lib/bitbucket_server/representation/base.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + class Base + attr_reader :raw + + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + def self.convert_timestamp(time_usec) + Time.at(time_usec / 1000) if time_usec.is_a?(Integer) + end + end + end +end diff --git a/lib/bitbucket_server/representation/comment.rb b/lib/bitbucket_server/representation/comment.rb new file mode 100644 index 00000000000..99b97a3b181 --- /dev/null +++ b/lib/bitbucket_server/representation/comment.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + # A general comment with the structure: + # "comment": { + # "author": { + # "active": true, + # "displayName": "root", + # "emailAddress": "stanhu+bitbucket@gitlab.com", + # "id": 1, + # "links": { + # "self": [ + # { + # "href": "http://localhost:7990/users/root" + # } + # ] + # }, + # "name": "root", + # "slug": "root", + # "type": "NORMAL" + # } + # } + # } + class Comment < Representation::Base + attr_reader :parent_comment + + CommentNode = Struct.new(:raw_comments, :parent) + + def initialize(raw, parent_comment: nil) + super(raw) + + @parent_comment = parent_comment + end + + def id + raw_comment['id'] + end + + def author_username + author['displayName'] + end + + def author_email + author['emailAddress'] + end + + def note + raw_comment['text'] + end + + def created_at + self.class.convert_timestamp(created_date) + end + + def updated_at + self.class.convert_timestamp(created_date) + end + + # Bitbucket Server supports the ability to reply to any comment + # and created multiple threads. It represents these as a linked list + # of comments within comments. For example: + # + # "comments": [ + # { + # "author" : ... + # "comments": [ + # { + # "author": ... + # + # Since GitLab only supports a single thread, we flatten all these + # comments into a single discussion. + def comments + @comments ||= flatten_comments + end + + private + + # In order to provide context for each reply, we need to track + # the parent of each comment. This method works as follows: + # + # 1. Insert the root comment into the workset. The root element is the current note. + # 2. For each node in the workset: + # a. Examine if it has replies to that comment. If it does, + # insert that node into the workset. + # b. Parse that note into a Comment structure and add it to a flat list. + def flatten_comments + comments = raw_comment['comments'] + workset = + if comments + [CommentNode.new(comments, self)] + else + [] + end + + all_comments = [] + + until workset.empty? + node = workset.pop + parent = node.parent + + node.raw_comments.each do |comment| + new_comments = comment.delete('comments') + current_comment = Comment.new({ 'comment' => comment }, parent_comment: parent) + all_comments << current_comment + workset << CommentNode.new(new_comments, current_comment) if new_comments + end + end + + all_comments + end + + def raw_comment + raw.fetch('comment', {}) + end + + def author + raw_comment['author'] + end + + def created_date + raw_comment['createdDate'] + end + + def updated_date + raw_comment['updatedDate'] + end + end + end +end diff --git a/lib/bitbucket_server/representation/pull_request.rb b/lib/bitbucket_server/representation/pull_request.rb new file mode 100644 index 00000000000..c3e927d8de7 --- /dev/null +++ b/lib/bitbucket_server/representation/pull_request.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + class PullRequest < Representation::Base + def author + raw.dig('author', 'user', 'name') + end + + def author_email + raw.dig('author', 'user', 'emailAddress') + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + case raw['state'] + when 'MERGED' + 'merged' + when 'DECLINED' + 'closed' + else + 'opened' + end + end + + def merged? + state == 'merged' + end + + def created_at + self.class.convert_timestamp(created_date) + end + + def updated_at + self.class.convert_timestamp(updated_date) + end + + def title + raw['title'] + end + + def source_branch_name + raw.dig('fromRef', 'id') + end + + def source_branch_sha + raw.dig('fromRef', 'latestCommit') + end + + def target_branch_name + raw.dig('toRef', 'id') + end + + def target_branch_sha + raw.dig('toRef', 'latestCommit') + end + + private + + def created_date + raw['createdDate'] + end + + def updated_date + raw['updatedDate'] + end + end + end +end diff --git a/lib/bitbucket_server/representation/pull_request_comment.rb b/lib/bitbucket_server/representation/pull_request_comment.rb new file mode 100644 index 00000000000..a2b3873a397 --- /dev/null +++ b/lib/bitbucket_server/representation/pull_request_comment.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + # An inline comment with the following structure that identifies + # the part of the diff: + # + # "commentAnchor": { + # "diffType": "EFFECTIVE", + # "fileType": "TO", + # "fromHash": "c5f4288162e2e6218180779c7f6ac1735bb56eab", + # "line": 1, + # "lineType": "ADDED", + # "orphaned": false, + # "path": "CHANGELOG.md", + # "toHash": "a4c2164330f2549f67c13f36a93884cf66e976be" + # } + # + # More details in https://docs.atlassian.com/bitbucket-server/rest/5.12.0/bitbucket-rest.html. + class PullRequestComment < Comment + def from_sha + comment_anchor['fromHash'] + end + + def to_sha + comment_anchor['toHash'] + end + + def to? + file_type == 'TO' + end + + def from? + file_type == 'FROM' + end + + def added? + line_type == 'ADDED' + end + + def removed? + line_type == 'REMOVED' + end + + # There are three line comment types: added, removed, or context. + # + # 1. An added type means a new line was inserted, so there is no old position. + # 2. A removed type means a line was removed, so there is no new position. + # 3. A context type means the line was unmodified, so there is both a + # old and new position. + def new_pos + return if removed? + return unless line_position + + line_position[1] + end + + def old_pos + return if added? + return unless line_position + + line_position[0] + end + + def file_path + comment_anchor.fetch('path') + end + + private + + def file_type + comment_anchor['fileType'] + end + + def line_type + comment_anchor['lineType'] + end + + # Each comment contains the following information about the diff: + # + # hunks: [ + # { + # segments: [ + # { + # "lines": [ + # { + # "commentIds": [ N ], + # "source": X, + # "destination": Y + # }, ... + # ] .... + # + # To determine the line position of a comment, we search all the lines + # entries until we find this comment ID. + def line_position + @line_position ||= diff_hunks.each do |hunk| + segments = hunk.fetch('segments', []) + segments.each do |segment| + lines = segment.fetch('lines', []) + lines.each do |line| + if line['commentIds']&.include?(id) + return [line['source'], line['destination']] + end + end + end + end + end + + def comment_anchor + raw.fetch('commentAnchor', {}) + end + + def diff + raw.fetch('diff', {}) + end + + def diff_hunks + diff.fetch('hunks', []) + end + end + end +end diff --git a/lib/bitbucket_server/representation/repo.rb b/lib/bitbucket_server/representation/repo.rb new file mode 100644 index 00000000000..6c494b79166 --- /dev/null +++ b/lib/bitbucket_server/representation/repo.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + class Repo < Representation::Base + def initialize(raw) + super(raw) + end + + def project_key + raw.dig('project', 'key') + end + + def project_name + raw.dig('project', 'name') + end + + def slug + raw['slug'] + end + + def browse_url + # The JSON reponse contains an array of 1 element. Not sure if there + # are cases where multiple links would be provided. + raw.dig('links', 'self').first.fetch('href') + end + + def clone_url + raw['links']['clone'].find { |link| link['name'].starts_with?('http') }.fetch('href') + end + + def description + project['description'] + end + + def full_name + "#{project_name}/#{name}" + end + + def issues_enabled? + true + end + + def name + raw['name'] + end + + def valid? + raw['scmId'] == 'git' + end + + def visibility_level + if project['public'] + Gitlab::VisibilityLevel::PUBLIC + else + Gitlab::VisibilityLevel::PRIVATE + end + end + + def project + raw['project'] + end + + def to_s + full_name + end + end + end +end diff --git a/lib/carrier_wave_string_file.rb b/lib/carrier_wave_string_file.rb index 6c848902e4a..c9a64d9e631 100644 --- a/lib/carrier_wave_string_file.rb +++ b/lib/carrier_wave_string_file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class CarrierWaveStringFile < StringIO def original_filename "" diff --git a/lib/constraints/feature_constrainer.rb b/lib/constraints/feature_constrainer.rb new file mode 100644 index 00000000000..cd246cf37a4 --- /dev/null +++ b/lib/constraints/feature_constrainer.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Constraints + class FeatureConstrainer + attr_reader :args + + def initialize(*args) + @args = args + end + + def matches?(_request) + Feature.enabled?(*args) + end + end +end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 87649c50424..8a3f8d2faaf 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Constraints class GroupUrlConstrainer def matches?(request) diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 32aea98f0f7..eadfbf7bc01 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Constraints class ProjectUrlConstrainer def matches?(request) diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 8afa04d29a4..e763569cb2e 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Constraints class UserUrlConstrainer def matches?(request) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index d5f85f9fcad..837b22c3082 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ContainerRegistry class Blob attr_reader :repository, :config diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 010ca1ec27b..c80f49f5ae0 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'faraday' require 'faraday_middleware' diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb index 589f9f4380a..740c0e13da0 100644 --- a/lib/container_registry/config.rb +++ b/lib/container_registry/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ContainerRegistry class Config attr_reader :tag, :blob, :data diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 61849a40383..9b2a61cdedc 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ContainerRegistry ## # Class responsible for extracting project and repository name from @@ -28,6 +30,7 @@ module ContainerRegistry @components ||= @path.split('/') end + # rubocop: disable CodeReuse/ActiveRecord def nodes raise InvalidRegistryPathError unless valid? @@ -35,17 +38,20 @@ module ContainerRegistry components.take(length).join('/') end end + # rubocop: enable CodeReuse/ActiveRecord def has_project? repository_project.present? end + # rubocop: disable CodeReuse/ActiveRecord def has_repository? return false unless has_project? repository_project.container_repositories .where(name: repository_name).any? end + # rubocop: enable CodeReuse/ActiveRecord def root_repository? @path == project_path diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index f90d711474a..523364ac7c7 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ContainerRegistry class Registry attr_reader :uri, :client, :path diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 728deea224f..ef41dc560c9 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -1,5 +1,9 @@ +# frozen_string_literal: true + module ContainerRegistry class Tag + include Gitlab::Utils::StrongMemoize + attr_reader :repository, :name delegate :registry, :client, to: :repository @@ -13,6 +17,10 @@ module ContainerRegistry manifest.present? end + def latest? + name == "latest" + end + def v1? manifest && manifest['schemaVersion'] == 1 end @@ -22,7 +30,9 @@ module ContainerRegistry end def manifest - @manifest ||= client.repository_manifest(repository.path, name) + strong_memoize(:manifest) do + client.repository_manifest(repository.path, name) + end end def path @@ -40,44 +50,54 @@ module ContainerRegistry end def digest - @digest ||= client.repository_tag_digest(repository.path, name) + strong_memoize(:digest) do + client.repository_tag_digest(repository.path, name) + end end def config_blob - return @config_blob if defined?(@config_blob) return unless manifest && manifest['config'] - @config_blob = repository.blob(manifest['config']) + strong_memoize(:config_blob) do + repository.blob(manifest['config']) + end end def config - return unless config_blob + return unless config_blob&.data - @config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data + strong_memoize(:config) do + ContainerRegistry::Config.new(self, config_blob) + end end def created_at return unless config - @created_at ||= DateTime.rfc3339(config['created']) + strong_memoize(:created_at) do + DateTime.rfc3339(config['created']) + end end def layers - return @layers if defined?(@layers) return unless manifest - layers = manifest['layers'] || manifest['fsLayers'] + strong_memoize(:layers) do + layers = manifest['layers'] || manifest['fsLayers'] - @layers = layers.map do |layer| - repository.blob(layer) + layers.map do |layer| + repository.blob(layer) + end end end + # rubocop: disable CodeReuse/ActiveRecord def total_size return unless layers layers.map(&:size).sum if v2? end + # rubocop: enable CodeReuse/ActiveRecord def delete return unless digest diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb index 1dd2855063d..7ba48ae9c79 100644 --- a/lib/declarative_policy.rb +++ b/lib/declarative_policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'declarative_policy/cache' require_dependency 'declarative_policy/condition' require_dependency 'declarative_policy/delegate_dsl' @@ -10,8 +12,6 @@ require_dependency 'declarative_policy/step' require_dependency 'declarative_policy/base' -require 'thread' - module DeclarativePolicy CLASS_CACHE_MUTEX = Mutex.new CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE @@ -21,7 +21,13 @@ module DeclarativePolicy cache = opts[:cache] || {} key = Cache.policy_key(user, subject) - cache[key] ||= class_for(subject).new(user, subject, opts) + cache[key] ||= + # to avoid deadlocks in multi-threaded environment when + # autoloading is enabled, we allow concurrent loads, + # https://gitlab.com/gitlab-org/gitlab-ce/issues/48263 + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + class_for(subject).new(user, subject, opts) + end end def class_for(subject) diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb index 47542194497..cd6e1606f22 100644 --- a/lib/declarative_policy/base.rb +++ b/lib/declarative_policy/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy class Base # A map of ability => list of rules together with :enable @@ -119,8 +121,8 @@ module DeclarativePolicy # a PolicyDsl which is used for registering the rule with # this class. PolicyDsl will call back into Base.enable_when, # Base.prevent_when, and Base.prevent_all_when. - def rule(&b) - rule = RuleDsl.new(self).instance_eval(&b) + def rule(&block) + rule = RuleDsl.new(self).instance_eval(&block) PolicyDsl.new(self, rule) end @@ -222,8 +224,8 @@ module DeclarativePolicy # computes the given ability and prints a helpful debugging output # showing which - def debug(ability, *a) - runner(ability).debug(*a) + def debug(ability, *args) + runner(ability).debug(*args) end desc "Unknown user" @@ -274,7 +276,7 @@ module DeclarativePolicy # # NOTE we can't use ||= here because the value might be the # boolean `false` - def cache(key, &b) + def cache(key) return @cache[key] if cached?(key) @cache[key] = yield diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb index 780d8f707bd..13006e56454 100644 --- a/lib/declarative_policy/cache.rb +++ b/lib/declarative_policy/cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy module Cache class << self diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb index 51c4a8b2bbe..b77f40b1093 100644 --- a/lib/declarative_policy/condition.rb +++ b/lib/declarative_policy/condition.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy # A Condition is the data structure that is created by the # `condition` declaration on DeclarativePolicy::Base. It is diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb index f544dffe888..67e3429b696 100644 --- a/lib/declarative_policy/delegate_dsl.rb +++ b/lib/declarative_policy/delegate_dsl.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy # Used when the name of a delegate is mentioned in # the rule DSL. @@ -7,10 +9,10 @@ module DeclarativePolicy @delegate_name = delegate_name end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - @rule_dsl.delegate(@delegate_name, m) + @rule_dsl.delegate(@delegate_name, msg) end end end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb index f11b6e9f730..96741c0478e 100644 --- a/lib/declarative_policy/policy_dsl.rb +++ b/lib/declarative_policy/policy_dsl.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy # The return value of a rule { ... } declaration. # Can call back to register rules with the containing @@ -15,8 +17,8 @@ module DeclarativePolicy @rule = rule end - def policy(&b) - instance_eval(&b) + def policy(&block) + instance_eval(&block) end def enable(*abilities) @@ -31,14 +33,14 @@ module DeclarativePolicy @context_class.prevent_all_when(@rule) end - def method_missing(m, *a, &b) - return super unless @context_class.respond_to?(m) + def method_missing(msg, *args, &block) + return super unless @context_class.respond_to?(msg) - @context_class.__send__(m, *a, &b) # rubocop:disable GitlabSecurity/PublicSend + @context_class.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end - def respond_to_missing?(m) - @context_class.respond_to?(m) || super + def respond_to_missing?(msg) + @context_class.respond_to?(msg) || super end end end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 5c214408dd0..239780d8626 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,8 +1,11 @@ -module DeclarativePolicy # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module DeclarativePolicy PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" class << self - def with_preferred_scope(scope, &b) + def with_preferred_scope(scope) Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY] yield ensure @@ -13,12 +16,12 @@ module DeclarativePolicy # rubocop:disable Naming/FileName Thread.current[PREFERRED_SCOPE_KEY] end - def user_scope(&b) - with_preferred_scope(:user, &b) + def user_scope(&block) + with_preferred_scope(:user, &block) end - def subject_scope(&b) - with_preferred_scope(:subject, &b) + def subject_scope(&block) + with_preferred_scope(:subject, &block) end def preferred_scope=(scope) diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb index e309244a3b3..f38f4f0a50f 100644 --- a/lib/declarative_policy/rule.rb +++ b/lib/declarative_policy/rule.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy module Rule # A Rule is the object that results from the `rule` declaration, @@ -8,8 +10,8 @@ module DeclarativePolicy # how that affects the actual ability decision - for that, a # `Step` is used. class Base - def self.make(*a) - new(*a).simplify + def self.make(*args) + new(*args).simplify end # true or false whether this rule passes. diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb index e948b7f2de1..85da7f261fa 100644 --- a/lib/declarative_policy/rule_dsl.rb +++ b/lib/declarative_policy/rule_dsl.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy # The DSL evaluation context inside rule { ... } blocks. # Responsible for creating and combining Rule objects. @@ -32,13 +34,13 @@ module DeclarativePolicy Rule::DelegatedCondition.new(delegate_name, condition) end - def method_missing(m, *a, &b) - return super unless a.empty? && !block_given? + def method_missing(msg, *args) + return super unless args.empty? && !block_given? - if @context_class.delegations.key?(m) - DelegateDsl.new(self, m) + if @context_class.delegations.key?(msg) + DelegateDsl.new(self, msg) else - cond(m.to_sym) + cond(msg.to_sym) end end end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb index 87f14b3b0d2..f739fe5e16e 100644 --- a/lib/declarative_policy/runner.rb +++ b/lib/declarative_policy/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy class Runner class State @@ -127,7 +129,7 @@ module DeclarativePolicy # # For each step, we yield the step object along with the computed score # for debugging purposes. - def steps_by_score(&b) + def steps_by_score flatten_steps! if @steps.size > 50 diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb index 3469fe9f991..c289c17cc19 100644 --- a/lib/declarative_policy/step.rb +++ b/lib/declarative_policy/step.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module DeclarativePolicy # This object represents one step in the runtime decision of whether # an ability is allowed. It contains a Rule and a context (instance diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb deleted file mode 100644 index cee664b8951..00000000000 --- a/lib/disable_email_interceptor.rb +++ /dev/null @@ -1,7 +0,0 @@ -# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails -class DisableEmailInterceptor - def self.delivering_email(message) - message.perform_deliveries = false - Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" - end -end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb deleted file mode 100644 index 3978a6d9fe4..00000000000 --- a/lib/email_template_interceptor.rb +++ /dev/null @@ -1,11 +0,0 @@ -# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails -class EmailTemplateInterceptor - def self.delivering_email(message) - # Remove HTML part if HTML emails are disabled. - unless Gitlab::CurrentSettings.html_emails_enabled - message.parts.delete_if do |part| - part.content_type.start_with?('text/html') - end - end - end -end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 515095af1c2..24fdcd6fbb1 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -1,74 +1,42 @@ -class EventFilter - attr_accessor :params - - class << self - def all - 'all' - end - - def push - 'push' - end - - def merged - 'merged' - end +# frozen_string_literal: true - def issue - 'issue' - end - - def comments - 'comments' - end - - def team - 'team' - end +class EventFilter + attr_accessor :filter + + ALL = 'all' + PUSH = 'push' + MERGED = 'merged' + ISSUE = 'issue' + COMMENTS = 'comments' + TEAM = 'team' + FILTERS = [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM].freeze + + def initialize(filter) + # Split using comma to maintain backward compatibility Ex/ "filter1,filter2" + filter = filter.to_s.split(',')[0].to_s + @filter = FILTERS.include?(filter) ? filter : ALL end - def initialize(params) - @params = if params - params.dup - else - [] # EventFilter.default_filter - end + def active?(key) + filter == key.to_s end + # rubocop: disable CodeReuse/ActiveRecord def apply_filter(events) - return events if params.blank? || params == EventFilter.all - - case params - when EventFilter.push + case filter + when PUSH events.where(action: Event::PUSHED) - when EventFilter.merged + when MERGED events.where(action: Event::MERGED) - when EventFilter.comments + when COMMENTS events.where(action: Event::COMMENTED) - when EventFilter.team + when TEAM events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) - when EventFilter.issue + when ISSUE events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) - end - end - - def options(key) - filter = params.dup - - if filter.include? key - filter.delete key - else - filter << key - end - - filter - end - - def active?(key) - if params.present? - params.include? key else - key == EventFilter.all + events end end + # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index 7b1533d0d32..c83cec9dc4a 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ExpandVariables class << self def expand(value, variables) diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a9b04c183ad..b2c8d46ede1 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Module providing methods for dealing with separating a tree-ish string and a # file path string when combined in a request parameter module ExtractsPath @@ -50,11 +52,13 @@ module ExtractsPath # branches and tags # Append a trailing slash if we only get a ref and no file path - id += '/' unless id.ends_with?('/') + unless id.ends_with?('/') + id = [id, '/'].join + end valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } - if valid_refs.length == 0 + if valid_refs.empty? # No exact ref match, so just try our best pair = id.match(%r{([^/]+)(.*)}).captures else @@ -106,11 +110,6 @@ module ExtractsPath # resolved (e.g., when a user inserts an invalid path or ref). # rubocop:disable Gitlab/ModuleWithInstanceVariables def assign_ref_vars - # assign allowed options - allowed_options = ["filter_ref"] - @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } - @options = HashWithIndifferentAccess.new(@options) - @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository @@ -139,16 +138,21 @@ module ExtractsPath def lfs_blob_ids blob_ids = tree.blobs.map(&:id) + + # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze + # the current Blob in order to determine if it's a LFS object + blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables + @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(@project.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables end private - # overriden in subclasses, do not remove + # overridden in subclasses, do not remove def get_id - id = params[:id] || params[:ref] - id += "/" + params[:path] unless params[:path].blank? - id + id = [params[:id] || params[:ref]] + id << "/" + params[:path] unless params[:path].blank? + id.join end def ref_names diff --git a/lib/feature.rb b/lib/feature.rb index 314ae224d90..e59cd70f822 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'flipper/adapters/active_record' require 'flipper/adapters/active_support_cache_store' @@ -28,22 +30,31 @@ class Feature end def persisted_names - if RequestStore.active? - RequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names - else - FlipperFeature.feature_names - end + Gitlab::SafeRequestStore[:flipper_persisted_names] ||= FlipperFeature.feature_names end def persisted?(feature) # Flipper creates on-memory features when asked for a not-yet-created one. # If we want to check if a feature has been actually set, we look for it # on the persisted features list. - persisted_names.include?(feature.name) + persisted_names.include?(feature.name.to_s) + end + + # use `default_enabled: true` to default the flag to being `enabled` + # unless set explicitly. The default is `disabled` + def enabled?(key, thing = nil, default_enabled: false) + feature = Feature.get(key) + + # If we're not default enabling the flag or the feature has been set, always evaluate. + # `persisted?` can potentially generate DB queries and also checks for inclusion + # in an array of feature names (177 at last count), possibly reducing performance by half. + # So we only perform the `persisted` check if `default_enabled: true` + !default_enabled || Feature.persisted?(feature) ? feature.enabled?(thing) : true end - def enabled?(key, thing = nil) - get(key).enabled?(thing) + def disabled?(key, thing = nil, default_enabled: false) + # we need to make different method calls to make it easy to mock / define expectations in test mode + thing.nil? ? !enabled?(key, default_enabled: default_enabled) : !enabled?(key, thing, default_enabled: default_enabled) end def enable(key, thing = true) @@ -63,8 +74,8 @@ class Feature end def flipper - if RequestStore.active? - RequestStore[:flipper] ||= build_flipper_instance + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance else @flipper ||= build_flipper_instance end @@ -91,4 +102,42 @@ class Feature expires_in: 1.hour) end end + + class Target + attr_reader :params + + def initialize(params) + @params = params + end + + def gate_specified? + %i(user project feature_group).any? { |key| params.key?(key) } + end + + def targets + [feature_group, user, project].compact + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def feature_group + return unless params.key?(:feature_group) + + Feature.group(params[:feature_group]) + end + # rubocop: enable CodeReuse/ActiveRecord + + def user + return unless params.key?(:user) + + UserFinder.new(params[:user]).find_by_username! + end + + def project + return unless params.key?(:project) + + Project.find_by_full_path(params[:project]) + end + end end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 69d981e8be9..70a145cd5bd 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class FileSizeValidator < ActiveModel::EachValidator MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze @@ -32,6 +34,7 @@ class FileSizeValidator < ActiveModel::EachValidator end end + # rubocop: disable CodeReuse/ActiveRecord def validate_each(record, attribute, value) raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base @@ -62,6 +65,7 @@ class FileSizeValidator < ActiveModel::EachValidator record.errors.add(attribute, MESSAGES[key], errors_options) end end + # rubocop: enable CodeReuse/ActiveRecord def help Helper.instance diff --git a/lib/flowdock/git.rb b/lib/flowdock/git.rb new file mode 100644 index 00000000000..f165ecfc1fa --- /dev/null +++ b/lib/flowdock/git.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +require 'flowdock' +require 'flowdock/git/builder' + +module Flowdock + class Git + TokenError = Class.new(StandardError) + + DEFAULT_PERMANENT_REFS = [ + Regexp.new('refs/heads/master') + ].freeze + + class << self + def post(ref, from, to, options = {}) + Git.new(ref, from, to, options).post + end + end + + def initialize(ref, from, to, options = {}) + raise TokenError.new("Flowdock API token not found") unless options[:token] + + @ref = ref + @from = from + @to = to + @options = options + @token = options[:token] + @commit_url = options[:commit_url] + @diff_url = options[:diff_url] + @repo_url = options[:repo_url] + @repo_name = options[:repo_name] + @permanent_refs = options.fetch(:permanent_refs, DEFAULT_PERMANENT_REFS) + end + + # Send git push notification to Flowdock + def post + messages.each do |message| + Flowdock::Client.new(flow_token: @token).post_to_thread(message) + end + end + + def repo + @options[:repo] + end + + private + + def messages + Git::Builder.new(repo: repo, + ref: @ref, + before: @from, + after: @to, + commit_url: @commit_url, + branch_url: @branch_url, + diff_url: @diff_url, + repo_url: @repo_url, + repo_name: @repo_name, + permanent_refs: @permanent_refs, + tags: tags + ).to_hashes + end + + # Flowdock tags attached to the push notification + def tags + Array(@options[:tags]).map { |tag| CGI.escape(tag) } + end + end +end diff --git a/lib/flowdock/git/builder.rb b/lib/flowdock/git/builder.rb new file mode 100644 index 00000000000..6f4428d1f42 --- /dev/null +++ b/lib/flowdock/git/builder.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true +module Flowdock + class Git + class Commit + def initialize(external_thread_id, thread, tags, commit) + @commit = commit + @external_thread_id = external_thread_id + @thread = thread + @tags = tags + end + + def to_hash + hash = { + external_thread_id: @external_thread_id, + event: "activity", + author: { + name: @commit[:author][:name], + email: @commit[:author][:email] + }, + title: title, + thread: @thread, + body: body + } + hash[:tags] = @tags if @tags + encode(hash) + end + + private + + def encode(hash) + return hash unless "".respond_to?(:encode) + + encode_as_utf8(hash) + end + + # This only works on Ruby 1.9 + def encode_as_utf8(obj) + if obj.is_a? Hash + obj.each_pair do |key, val| + encode_as_utf8(val) + end + elsif obj.is_a?(Array) + obj.each do |val| + encode_as_utf8(val) + end + elsif obj.is_a?(String) && obj.encoding != Encoding::UTF_8 + unless obj.force_encoding("UTF-8").valid_encoding? + obj.force_encoding("ISO-8859-1").encode!(Encoding::UTF_8, invalid: :replace, undef: :replace) + end + end + end + + def body + content = @commit[:message][first_line.size..-1] + content.strip! if content + "<pre>#{content}</pre>" unless content.empty? + end + + def first_line + @first_line ||= (@commit[:message].split("\n")[0] || @commit[:message]) + end + + def title + commit_id = @commit[:id][0, 7] + if @commit[:url] + "<a href=\"#{@commit[:url]}\">#{commit_id}</a> #{message_title}" + else + "#{commit_id} #{message_title}" + end + end + + def message_title + CGI.escape_html(first_line.strip) + end + end + + # Class used to build Git payload + class Builder + include ::Gitlab::Utils::StrongMemoize + + def initialize(opts) + @repo = opts[:repo] + @ref = opts[:ref] + @before = opts[:before] + @after = opts[:after] + @opts = opts + end + + def commits + @repo.commits_between(@before, @after).map do |commit| + { + url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil, + id: commit.sha, + message: commit.message, + author: { + name: commit.author_name, + email: commit.author_email + } + } + end + end + + def ref_name + @ref.to_s.sub(%r{\Arefs/(heads|tags)/}, '') + end + + def to_hashes + commits.map do |commit| + Commit.new(external_thread_id, thread, @opts[:tags], commit).to_hash + end + end + + private + + def thread + @thread ||= { + title: thread_title, + external_url: @opts[:repo_url] + } + end + + def permanent? + strong_memoize(:permanent) do + @opts[:permanent_refs].any? { |regex| regex.match(@ref) } + end + end + + def thread_title + action = "updated" if permanent? + type = @ref =~ %r(^refs/heads/) ? "branch" : "tag" + + [@opts[:repo_name], type, ref_name, action].compact.join(" ") + end + + def external_thread_id + @external_thread_id ||= + if permanent? + SecureRandom.hex + else + @ref + end + end + end + end +end diff --git a/lib/forever.rb b/lib/forever.rb index 7df17912544..0a37118fe68 100644 --- a/lib/forever.rb +++ b/lib/forever.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class Forever POSTGRESQL_DATE = DateTime.new(3000, 1, 1) MYSQL_DATE = DateTime.new(2038, 01, 19) diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb index 91175b49c79..15cdd25e711 100644 --- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'rails/generators' module Rails diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 605e93022e7..7b238623418 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitaly class Server def self.all @@ -22,6 +24,18 @@ module Gitaly server_version == Gitlab::GitalyClient.expected_server_version end + def read_writeable? + readable? && writeable? + end + + def readable? + storage_status&.readable + end + + def writeable? + storage_status&.writeable + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -30,13 +44,17 @@ module Gitaly private + def storage_status + @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } + end + def info @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info - rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded + rescue GRPC::Unavailable, GRPC::DeadlineExceeded # This will show the server as being out of date - Gitaly::ServerInfoResponse.new(git_version: '', server_version: '') + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index a129746e2c6..e073450283b 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'gitlab/popen' module Gitlab @@ -5,12 +7,16 @@ module Gitlab Pathname.new(File.expand_path('..', __dir__)) end - def self.config - Settings + def self.version_info + Gitlab::VersionInfo.parse(Gitlab::VERSION) + end + + def self.pre_release? + VERSION.include?('pre') end - def self.migrations_hash - @_migrations_hash ||= Digest::MD5.hexdigest(ActiveRecord::Migrator.get_all_versions.to_s) + def self.config + Settings end def self.revision @@ -33,6 +39,7 @@ module Gitlab APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} VERSION = File.read(root.join("VERSION")).strip.freeze + INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze def self.com? # Check `gl_subdomain?` as well to keep parity with gitlab.com @@ -50,4 +57,12 @@ module Gitlab def self.dev_env_or_com? Rails.env.development? || org? || com? end + + def self.process_name + return 'sidekiq' if Sidekiq.server? + return 'console' if defined?(Rails::Console) + return 'test' if Rails.env.test? + + 'web' + end end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 7127948cf00..ec090aea784 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitlab::Access module # # Define allowed roles that can be used @@ -7,12 +9,14 @@ module Gitlab module Access AccessDeniedError = Class.new(StandardError) - NO_ACCESS = 0 - GUEST = 10 - REPORTER = 20 - DEVELOPER = 30 - MASTER = 40 - OWNER = 50 + NO_ACCESS = 0 + GUEST = 10 + REPORTER = 20 + DEVELOPER = 30 + MAINTAINER = 40 + # @deprecated + MASTER = MAINTAINER + OWNER = 50 # Branch protection settings PROTECTION_NONE = 0 @@ -29,10 +33,10 @@ module Gitlab def options { - "Guest" => GUEST, - "Reporter" => REPORTER, - "Developer" => DEVELOPER, - "Master" => MASTER + "Guest" => GUEST, + "Reporter" => REPORTER, + "Developer" => DEVELOPER, + "Maintainer" => MAINTAINER } end @@ -44,10 +48,10 @@ module Gitlab def sym_options { - guest: GUEST, - reporter: REPORTER, - developer: DEVELOPER, - master: MASTER + guest: GUEST, + reporter: REPORTER, + developer: DEVELOPER, + maintainer: MAINTAINER } end @@ -57,10 +61,10 @@ module Gitlab def protection_options { - "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, - "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Masters can push to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Both developers and masters can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, but masters can. No-one can force push or delete the branch." => PROTECTION_FULL + "Not protected: Both developers and maintainers can push new commits, force push, or delete the branch." => PROTECTION_NONE, + "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch." => PROTECTION_DEV_CAN_MERGE, + "Partially protected: Both developers and maintainers can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but maintainers can. No-one can force push or delete the branch." => PROTECTION_FULL } end diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb new file mode 100644 index 00000000000..f039e5c011f --- /dev/null +++ b/lib/gitlab/access/branch_protection.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Access + # A wrapper around Integer based branch protection levels. + # + # This wrapper can be used to work with branch protection levels without + # having to directly refer to the constants. For example, instead of this: + # + # if access_level == Gitlab::Access::PROTECTION_DEV_CAN_PUSH + # ... + # end + # + # You can write this instead: + # + # protection = BranchProtection.new(access_level) + # + # if protection.developer_can_push? + # ... + # end + class BranchProtection + attr_reader :level + + # @param [Integer] level The branch protection level as an Integer. + def initialize(level) + @level = level + end + + def any? + level != PROTECTION_NONE + end + + def developer_can_push? + level == PROTECTION_DEV_CAN_PUSH + end + + def developer_can_merge? + level == PROTECTION_DEV_CAN_MERGE + end + end + end +end diff --git a/lib/gitlab/action_rate_limiter.rb b/lib/gitlab/action_rate_limiter.rb index 4cd3bdefda3..c442211e073 100644 --- a/lib/gitlab/action_rate_limiter.rb +++ b/lib/gitlab/action_rate_limiter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # This class implements a simple rate limiter that can be used to throttle # certain actions. Unlike Rack Attack and Rack::Throttle, which operate at diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index 45c2b01dd8f..4518c8a862c 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Allowable def can?(*args) diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb index dddcb2538f9..5edec8b3efe 100644 --- a/lib/gitlab/app_logger.rb +++ b/lib/gitlab/app_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class AppLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 62c41801d75..df8f0470063 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'asciidoctor' require 'asciidoctor/converter/html5' require "asciidoctor-plantuml" diff --git a/lib/gitlab/audit_json_logger.rb b/lib/gitlab/audit_json_logger.rb new file mode 100644 index 00000000000..12e0645f3e4 --- /dev/null +++ b/lib/gitlab/audit_json_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class AuditJsonLogger < Gitlab::JsonLogger + def self.file_name_noext + 'audit_json' + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0f7a7b0ce8d..7aa02009aa0 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth MissingPersonalAccessTokenError = Class.new(StandardError) @@ -14,23 +16,8 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self - def omniauth_customized_providers - @omniauth_customized_providers ||= %w[bitbucket jwt] - end - - def omniauth_setup_providers(provider_names) - provider_names.each do |provider| - omniauth_setup_a_provider(provider) - end - end - - def omniauth_setup_a_provider(provider) - case provider - when 'kerberos' - require 'omniauth-kerberos' - when *omniauth_customized_providers - require_dependency "omni_auth/strategies/#{provider}" - end + def omniauth_enabled? + Gitlab.config.omniauth.enabled end def find_for_git_client(login, password, project:, ip:) @@ -151,6 +138,7 @@ module Gitlab Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) end + # rubocop: disable CodeReuse/ActiveRecord def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) @@ -161,11 +149,12 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def personal_access_token_check(password) return unless password.present? - token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) + token = PersonalAccessTokensFinder.new(state: 'active').find_by_token(password) if token && valid_scoped_token?(token, available_scopes) Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) @@ -192,6 +181,7 @@ module Gitlab end.uniq end + # rubocop: disable CodeReuse/ActiveRecord def deploy_token_check(login, password) return unless password.present? @@ -207,8 +197,9 @@ module Gitlab Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) end end + # rubocop: enable CodeReuse/ActiveRecord - def lfs_token_check(login, password, project) + def lfs_token_check(login, encoded_token, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) actor = @@ -231,7 +222,7 @@ module Gitlab read_authentication_abilities end - if Devise.secure_compare(token_handler.token, password) + if token_handler.token_valid?(encoded_token) Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities) end end @@ -240,7 +231,7 @@ module Gitlab return unless login == 'gitlab-ci-token' return unless password - build = ::Ci::Build.running.find_by_token(password) + build = find_build_by_token(password) return unless build return unless build.project.builds_enabled? @@ -301,6 +292,12 @@ module Gitlab REGISTRY_SCOPES end + + private + + def find_build_by_token(token) + ::Ci::Build.running.find_by_token(token) + end end end end diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb new file mode 100644 index 00000000000..558628b5422 --- /dev/null +++ b/lib/gitlab/auth/activity.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + ## + # Metrics and logging for user authentication activity. + # + class Activity + extend Gitlab::Utils::StrongMemoize + + COUNTERS = { + user_authenticated: 'Counter of successful authentication events', + user_unauthenticated: 'Counter of authentication failures', + user_not_found: 'Counter of failed log-ins when user is unknown', + user_password_invalid: 'Counter of failed log-ins with invalid password', + user_session_override: 'Counter of manual log-ins and sessions overrides', + user_session_destroyed: 'Counter of user sessions being destroyed', + user_two_factor_authenticated: 'Counter of two factor authentications', + user_sessionless_authentication: 'Counter of sessionless authentications', + user_blocked: 'Counter of sign in attempts when user is blocked' + }.freeze + + def initialize(opts) + @opts = opts + end + + def user_authentication_failed! + self.class.user_unauthenticated_counter_increment! + + case @opts[:message] + when :not_found_in_database + self.class.user_not_found_counter_increment! + when :invalid + self.class.user_password_invalid_counter_increment! + end + end + + def user_authenticated! + self.class.user_authenticated_counter_increment! + end + + def user_session_override! + self.class.user_session_override_counter_increment! + + case @opts[:message] + when :two_factor_authenticated + self.class.user_two_factor_authenticated_counter_increment! + when :sessionless_sign_in + self.class.user_sessionless_authentication_counter_increment! + end + end + + def user_blocked! + self.class.user_blocked_counter_increment! + end + + def user_session_destroyed! + self.class.user_session_destroyed_counter_increment! + end + + def self.each_counter + COUNTERS.each_pair do |metric, description| + yield "#{metric}_counter", metric, description + end + end + + each_counter do |counter, metric, description| + define_singleton_method(counter) do + strong_memoize(counter) do + Gitlab::Metrics.counter("gitlab_auth_#{metric}_total".to_sym, description) + end + end + + define_singleton_method("#{counter}_increment!") do + public_send(counter).increment # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb index 7609a7b04f6..50712d7eac2 100644 --- a/lib/gitlab/auth/blocked_user_tracker.rb +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -2,35 +2,19 @@ module Gitlab module Auth class BlockedUserTracker - ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters' - - def self.log_if_user_blocked(env) - message = env.dig('warden.options', :message) - - # Devise calls User#active_for_authentication? on the User model and then - # throws an exception to Warden with User#inactive_message: - # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8 - # - # Since Warden doesn't pass the user record to the failure handler, we - # need to do a database lookup with the username. We can limit the - # lookups to happen when the user was blocked by checking the inactive - # message passed along by Warden. - return unless message == User::BLOCKED_MESSAGE - - # Check for either LDAP or regular GitLab account logins - login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') || - env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') - - return unless login.present? - - user = User.by_login(login) + def initialize(user, auth) + @user = user + @auth = auth + end - return unless user&.blocked? + def log_activity! + return unless @user.blocked? - Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}") - SystemHooksService.new.execute_hooks_for(user, :failed_login) + Gitlab::AppLogger.info <<~INFO + "Failed login for blocked user: user=#{@user.username} ip=#{@auth.request.ip}") + INFO - true + SystemHooksService.new.execute_hooks_for(@user, :failed_login) rescue TypeError end end diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index 1234ace0334..c0dc2b0875f 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # These calls help to authenticate to OAuth provider by providing username and password # diff --git a/lib/gitlab/auth/ip_rate_limiter.rb b/lib/gitlab/auth/ip_rate_limiter.rb index e6173d45af3..81e616fa20a 100644 --- a/lib/gitlab/auth/ip_rate_limiter.rb +++ b/lib/gitlab/auth/ip_rate_limiter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth class IpRateLimiter diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 865185eb5db..c875bba4bcb 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # LDAP authorization model # # * Check if we are allowed access (not blocked) @@ -19,8 +21,10 @@ module Gitlab # Whether user is allowed, or not, we should update # permissions to keep things clean if access.allowed? - access.update_user - Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute + unless Gitlab::Database.read_only? + access.update_user + Users::UpdateService.new(user, user: user, last_credential_check_at: Time.now).execute + end true else @@ -60,6 +64,12 @@ module Gitlab false end + def update_user + # no-op in CE + end + + private + def adapter @adapter ||= Gitlab::Auth::LDAP::Adapter.new(provider) end @@ -68,28 +78,28 @@ module Gitlab Gitlab::Auth::LDAP::Config.new(provider) end - def find_ldap_user - Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter) - end - def ldap_user return unless provider @ldap_user ||= find_ldap_user end + def find_ldap_user + Gitlab::Auth::LDAP::Person.find_by_dn(ldap_identity.extern_uid, adapter) + end + def block_user(user, reason) user.ldap_block if provider Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) else Gitlab::AppLogger.info( "Account is not provided by LDAP, " \ - "blocking Gitlab user \"#{user.name}\" (#{user.email})" + "blocking GitLab user \"#{user.name}\" (#{user.email})" ) end end @@ -99,13 +109,9 @@ module Gitlab Gitlab::AppLogger.info( "LDAP account \"#{ldap_identity.extern_uid}\" #{reason}, " \ - "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + "unblocking GitLab user \"#{user.name}\" (#{user.email})" ) end - - def update_user - # no-op in CE - end end end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 82ff1e77e5c..15b9d5ad6e9 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module LDAP @@ -28,14 +30,7 @@ module Gitlab def users(fields, value, limit = nil) options = user_options(Array(fields), value, limit) - - entries = ldap_search(options).select do |entry| - entry.respond_to? config.uid - end - - entries.map do |entry| - Gitlab::Auth::LDAP::Person.new(entry, provider) - end + users_search(options) end def user(*args) @@ -88,6 +83,16 @@ module Gitlab SEARCH_RETRY_FACTOR[retry_number] * config.timeout end + def users_search(options) + entries = ldap_search(options).select do |entry| + entry.respond_to? config.uid + end + + entries.map do |entry| + Gitlab::Auth::LDAP::Person.new(entry, provider) + end + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb index ac5c14d374d..83fdc8a8c76 100644 --- a/lib/gitlab/auth/ldap/auth_hash.rb +++ b/lib/gitlab/auth/ldap/auth_hash.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Class to parse and transform the info provided by omniauth # module Gitlab diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index 7c134fb6438..174e81dd603 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # These calls help to authenticate to LDAP by providing username and password # # Since multiple LDAP servers are supported, it will loop through all of them diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index d4415eaa6dc..7ceb96f502b 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Load a specific server configuration module Gitlab module Auth diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index 1fa5338f5a6..5df914aa367 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -1,4 +1,5 @@ # -*- ruby encoding: utf-8 -*- +# frozen_string_literal: true # Based on the `ruby-net-ldap` gem's `Net::LDAP::DN` # diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb index ef0a695742b..d0e5f24d203 100644 --- a/lib/gitlab/auth/ldap/ldap_connection_error.rb +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module LDAP diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 8dfae3ee541..48d134f91b0 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module LDAP @@ -96,9 +98,7 @@ module Gitlab private - def entry - @entry - end + attr_reader :entry def config @config ||= Gitlab::Auth::LDAP::Config.new(provider) diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 922d0567d99..9c71671f409 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # LDAP extension for User model # # * Find or create user from omniauth.auth data @@ -11,11 +13,13 @@ module Gitlab extend ::Gitlab::Utils::Override class << self + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider(uid, provider) identity = ::Identity.with_extern_uid(provider, uid).take identity && identity.user end + # rubocop: enable CodeReuse/ActiveRecord end def save diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index ed8fba94305..36fc8061d92 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Class to parse and transform the info provided by omniauth # module Gitlab @@ -78,7 +80,7 @@ module Gitlab end # Get the first part of the email address (before @) - # In addtion in removes illegal characters + # In addition in removes illegal characters def generate_username(email) email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb index d4e7f35c857..5f008678bd1 100644 --- a/lib/gitlab/auth/o_auth/authentication.rb +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # These calls help to authenticate to OAuth provider by providing username and password # diff --git a/lib/gitlab/auth/o_auth/identity_linker.rb b/lib/gitlab/auth/o_auth/identity_linker.rb index de92d7a214d..e69c2bb54dc 100644 --- a/lib/gitlab/auth/o_auth/identity_linker.rb +++ b/lib/gitlab/auth/o_auth/identity_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module OAuth diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 5fb61ffe00d..9fdf3324db3 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module OAuth @@ -29,8 +31,9 @@ module Gitlab def self.enabled?(name) return true if name == 'database' + return true if self.ldap_provider?(name) && providers.include?(name.to_sym) - providers.include?(name.to_sym) + Gitlab::Auth.omniauth_enabled? && providers.include?(name.to_sym) end def self.ldap_provider?(name) diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb index 8f2b4d58552..4925b107042 100644 --- a/lib/gitlab/auth/o_auth/session.rb +++ b/lib/gitlab/auth/o_auth/session.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # :nocov: module Gitlab module Auth diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 6c5d0788a0a..f38c5d57c44 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # OAuth extension for User model # # * Find GitLab user based on omniauth uid and provider @@ -44,11 +46,11 @@ module Gitlab gl_user.block if block_after_save - log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" + log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" - return self, e.record.errors + [self, e.record.errors] end def gl_user @@ -74,6 +76,10 @@ module Gitlab gl_user end + def bypass_two_factor? + false + end + protected def should_save? @@ -108,11 +114,13 @@ module Gitlab build_new_user end + # rubocop: disable CodeReuse/ActiveRecord def find_by_email return unless auth_hash.has_attribute?(:email) ::User.find_by(email: auth_hash.email.downcase) end + # rubocop: enable CodeReuse/ActiveRecord def auto_link_ldap_user? Gitlab.config.omniauth.auto_link_ldap_user @@ -176,10 +184,12 @@ module Gitlab @auth_hash = AuthHash.new(auth_hash) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_uid_and_provider identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take identity&.user end + # rubocop: enable CodeReuse/ActiveRecord def build_new_user user_params = user_attributes.merge(skip_confirmation: true) diff --git a/lib/gitlab/auth/omniauth_identity_linker_base.rb b/lib/gitlab/auth/omniauth_identity_linker_base.rb index f79ce6bb809..253445570f2 100644 --- a/lib/gitlab/auth/omniauth_identity_linker_base.rb +++ b/lib/gitlab/auth/omniauth_identity_linker_base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth class OmniauthIdentityLinkerBase @@ -33,11 +35,13 @@ module Gitlab @changed = identity.save end + # rubocop: disable CodeReuse/ActiveRecord def identity @identity ||= current_user.identities .with_extern_uid(provider, uid) .first_or_initialize(extern_uid: uid) end + # rubocop: enable CodeReuse/ActiveRecord def provider oauth['provider'] diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 66de52506ce..176766d1a8b 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Use for authentication only, in particular for Rack::Attack. # Does not perform authorization of scopes, etc. module Gitlab @@ -11,12 +13,18 @@ module Gitlab @request = request end - def user - find_sessionless_user || find_user_from_warden + def user(request_formats) + request_formats.each do |format| + user = find_sessionless_user(format) + + return user if user + end + + find_user_from_warden end - def find_sessionless_user - find_user_from_access_token || find_user_from_feed_token + def find_sessionless_user(request_format) + find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format) rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 00cdc94a9ef..78fa25c5516 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -1,4 +1,7 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module Auth Result = Struct.new(:actor, :project, :type, :authentication_abilities) do def ci?(for_project) diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index c345a7e3f6c..1af9fa40c3a 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module Saml @@ -6,6 +8,17 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) end + def authn_context + response_object = auth_hash.extra[:response_object] + return nil if response_object.blank? + + document = response_object.decrypted_document + document ||= response_object.document + return nil if document.blank? + + extract_authn_context(document) + end + private def get_raw(key) @@ -13,6 +26,10 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end + + def extract_authn_context(document) + REXML::XPath.first(document, "//*[name()='saml:AuthnStatement' or name()='saml2:AuthnStatement']/*[name()='saml:AuthnContext' or name()='saml2:AuthnContext']/*[name()='saml:AuthnContextClassRef' or name()='saml2:AuthnContextClassRef']/text()").to_s + end end end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 5fa9581f837..8cb999f50d4 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module Saml @@ -7,6 +9,10 @@ module Gitlab Gitlab::Auth::OAuth::Provider.config_for('saml') end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end + def groups options[:groups_attribute] end diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb index 7e4b191d512..ae0d6dded4e 100644 --- a/lib/gitlab/auth/saml/identity_linker.rb +++ b/lib/gitlab/auth/saml/identity_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth module Saml diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index b8c84c37cd5..ec95bc46791 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # SAML extension for User model # # * Find GitLab user based on SAML uid and provider @@ -34,6 +36,10 @@ module Gitlab gl_user.changed? || gl_user.identities.any?(&:changed?) end + def bypass_two_factor? + saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) + end + protected def saml_config diff --git a/lib/gitlab/auth/too_many_ips.rb b/lib/gitlab/auth/too_many_ips.rb index ed862791551..ee4d80e6b89 100644 --- a/lib/gitlab/auth/too_many_ips.rb +++ b/lib/gitlab/auth/too_many_ips.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth class TooManyIps < StandardError diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index baa1f802d8a..31dd61ae6cf 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth class UniqueIpsLimiter diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index 1893cb001b2..fd09fe76c02 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth class UserAccessDeniedReason diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index c7993665421..a5efe33bdc6 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Auth AuthenticationError = Class.new(StandardError) @@ -5,6 +7,7 @@ module Gitlab TokenNotFoundError = Class.new(AuthenticationError) ExpiredError = Class.new(AuthenticationError) RevokedError = Class.new(AuthenticationError) + ImpersonationDisabled = Class.new(AuthenticationError) UnauthorizedError = Class.new(AuthenticationError) class InsufficientScopeError < AuthenticationError @@ -25,8 +28,8 @@ module Gitlab current_request.env['warden']&.authenticate if verified_request? end - def find_user_from_feed_token - return unless rss_request? || ics_request? + def find_user_from_feed_token(request_format) + return unless valid_rss_format?(request_format) # NOTE: feed_token was renamed from rss_token but both needs to be supported because # users might have already added the feed to their RSS reader before the rename @@ -36,6 +39,17 @@ module Gitlab User.find_by_feed_token(token) || raise(UnauthorizedError) end + # We only allow Private Access Tokens with `api` scope to be used by web + # requests on RSS feeds or ICS files for backwards compatibility. + # It is also used by GraphQL/API requests. + def find_user_from_web_access_token(request_format) + return unless access_token && valid_web_access_format?(request_format) + + validate_access_token!(scopes: [:api]) + + access_token.user || raise(UnauthorizedError) + end + def find_user_from_access_token return unless access_token @@ -54,6 +68,8 @@ module Gitlab raise ExpiredError when AccessTokenValidationService::REVOKED raise RevokedError + when AccessTokenValidationService::IMPERSONATION_DISABLED + raise ImpersonationDisabled end end @@ -79,7 +95,7 @@ module Gitlab return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` - PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) + PersonalAccessToken.find_by_token(token) || raise(UnauthorizedError) end def find_oauth_access_token @@ -107,6 +123,26 @@ module Gitlab @current_request ||= ensure_action_dispatch_request(request) end + def valid_web_access_format?(request_format) + case request_format + when :rss + rss_request? + when :ics + ics_request? + when :api + api_request? + end + end + + def valid_rss_format?(request_format) + case request_format + when :rss + rss_request? + when :ics + ics_request? + end + end + def rss_request? current_request.path.ends_with?('.atom') || current_request.format.atom? end @@ -114,6 +150,10 @@ module Gitlab def ics_request? current_request.path.ends_with?('.ics') || current_request.format.ics? end + + def api_request? + current_request.path.starts_with?("/api/") + end end end end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d3f66877672..5251e0fadf9 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration def self.queue @@ -14,11 +16,18 @@ module Gitlab # re-raises the exception. # # steal_class - The name of the class for which to steal jobs. - def self.steal(steal_class) - enqueued = Sidekiq::Queue.new(self.queue) - scheduled = Sidekiq::ScheduledSet.new + def self.steal(steal_class, retry_dead_jobs: false) + queues = [ + Sidekiq::ScheduledSet.new, + Sidekiq::Queue.new(self.queue) + ] + + if retry_dead_jobs + queues << Sidekiq::RetrySet.new + queues << Sidekiq::DeadSet.new + end - [scheduled, enqueued].each do |queue| + queues.each do |queue| queue.each do |job| migration_class, migration_args = job.args @@ -46,7 +55,24 @@ module Gitlab # arguments - The arguments to pass to the background migration's "perform" # method. def self.perform(class_name, arguments) - const_get(class_name).new.perform(*arguments) + migration_class_for(class_name).new.perform(*arguments) + end + + def self.exists?(migration_class) + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new + + [enqueued, scheduled].each do |queue| + queue.each do |job| + return true if job.queue == self.queue && job.args.first == migration_class + end + end + + false + end + + def self.migration_class_for(class_name) + const_get(class_name) end end end diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index d5cf9e0d53a..cb2bdea755c 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation -# rubocop:disable Metrics/LineLength module Gitlab module BackgroundMigration diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb new file mode 100644 index 00000000000..92096e29ef1 --- /dev/null +++ b/lib/gitlab/background_migration/archive_legacy_traces.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ArchiveLegacyTraces + def perform(start_id, stop_id) + # This background migration directly refers to ::Ci::Build model which is defined in application code. + # In general, migration code should be isolated as much as possible in order to be idempotent. + # However, `archive!` method is too complicated to be replicated by coping its subsequent code. + # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 + ::Ci::Build.finished.without_archived_trace + .where(id: start_id..stop_id).find_each do |build| + begin + build.trace.archive! + rescue => e + Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb new file mode 100644 index 00000000000..a6194616663 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will fill the project_repositories table for projects that + # are on hashed storage and an entry is is missing in this table. + class BackfillHashedProjectRepositories < BackfillProjectRepositories + private + + def projects + Project.on_hashed_storage + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb new file mode 100644 index 00000000000..6dc92672929 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_legacy_project_repositories.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will fill the project_repositories table for projects that + # are on legacy storage and an entry is is missing in this table. + class BackfillLegacyProjectRepositories < BackfillProjectRepositories + private + + def projects + Project.with_parent.on_legacy_storage + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb new file mode 100644 index 00000000000..29fa0f18448 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This module is used to write the full path of all projects to + # the git repository config file. + # Storing the full project path in the git config allows admins to + # easily identify a project when it is using hashed storage. + module BackfillProjectFullpathInRepoConfig + OrphanedNamespaceError = Class.new(StandardError) + + module Storage + # Class that returns the disk path for a project using hashed storage + class HashedProject + attr_accessor :project + + ROOT_PATH_PREFIX = '@hashed' + + def initialize(project) + @project = project + end + + def disk_path + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id + end + end + + # Class that returns the disk path for a project using legacy storage + class LegacyProject + attr_accessor :project + + def initialize(project) + @project = project + end + + def disk_path + project.full_path + end + end + end + + # Concern used by Project and Namespace to determine the full + # route to the project + module Routable + extend ActiveSupport::Concern + + def full_path + @full_path ||= build_full_path + end + + def build_full_path + return path unless has_parent? + + raise OrphanedNamespaceError if parent.nil? + + parent.full_path + '/' + path + end + + def has_parent? + read_attribute(association(:parent).reflection.foreign_key) + end + end + + # Class used to interact with repository using Gitaly + class Repository + attr_reader :storage + + def initialize(storage, relative_path) + @storage = storage + @relative_path = relative_path + end + + def gitaly_repository + Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path) + end + end + + # Namespace can be a user or group. It can be the root or a + # child of another namespace. + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + self.inheritance_column = nil + + include Routable + + belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + has_many :projects, inverse_of: :parent + has_many :namespaces, inverse_of: :parent + end + + # Project is where the repository (etc.) is stored + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include Routable + include EachBatch + + FULLPATH_CONFIG_KEY = 'gitlab.fullpath' + + belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + delegate :disk_path, to: :storage + + def add_fullpath_config + entries = { FULLPATH_CONFIG_KEY => full_path } + + repository_service.set_config(entries) + end + + def remove_fullpath_config + repository_service.delete_config([FULLPATH_CONFIG_KEY]) + end + + def cleanup_repository + repository_service.cleanup + end + + def storage + @storage ||= + if hashed_storage? + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end + + def hashed_storage? + self.storage_version && self.storage_version >= 1 + end + + def repository + @repository ||= Repository.new(repository_storage, disk_path + '.git') + end + + def repository_service + @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository) + end + end + + # Base class for Up and Down migration classes + class BackfillFullpathMigration + RETRY_DELAY = 15.minutes + MAX_RETRIES = 2 + + # Base class for retrying one project + class BaseRetryOne + def perform(project_id, retry_count) + project = Project.find(project_id) + + return unless project + + migration_class.new.safe_perform_one(project, retry_count) + end + end + + def perform(start_id, end_id) + Project.includes(:parent).where(id: start_id..end_id).each do |project| + safe_perform_one(project) + end + end + + def safe_perform_one(project, retry_count = 0) + perform_one(project) + rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError + nil + rescue GRPC::BadStatus + schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES + end + + def schedule_retry(project, retry_count) + BackgroundMigrationWorker.perform_in(RETRY_DELAY, self.class::RetryOne.name, [project.id, retry_count]) + end + end + + # Class to add the fullpath to the git repo config + class Up < BackfillFullpathMigration + # Class used to retry + class RetryOne < BaseRetryOne + def migration_class + Up + end + end + + def perform_one(project) + project.cleanup_repository + project.add_fullpath_config + end + end + + # Class to rollback adding the fullpath to the git repo config + class Down < BackfillFullpathMigration + # Class used to retry + class RetryOne < BaseRetryOne + def migration_class + Down + end + end + + def perform_one(project) + project.cleanup_repository + project.remove_fullpath_config + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb new file mode 100644 index 00000000000..c8d83cc1803 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will create fill the project_repositories table + # for projects an entry is is missing in this table. + class BackfillProjectRepositories + OrphanedNamespaceError = Class.new(StandardError) + + # Shard model + class Shard < ActiveRecord::Base + self.table_name = 'shards' + end + + # Class that will find or create the shard by name. + # There is only a small set of shards, which would + # not change quickly, so look them up from memory + # instead of hitting the DB each time. + class ShardFinder + def find_shard_id(name) + shard_id = shards.fetch(name, nil) + return shard_id if shard_id.present? + + Shard.transaction(requires_new: true) do + create!(name) + end + rescue ActiveRecord::RecordNotUnique + reload! + retry + end + + private + + def create!(name) + Shard.create!(name: name).tap { |shard| @shards[name] = shard.id } + end + + def shards + @shards ||= reload! + end + + def reload! + @shards = Hash[*Shard.all.map { |shard| [shard.name, shard.id] }.flatten] + end + end + + module Storage + # Class that returns the disk path for a project using hashed storage + class HashedProject + attr_accessor :project + + ROOT_PATH_PREFIX = '@hashed' + + def initialize(project) + @project = project + end + + def disk_path + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) + end + end + + # Class that returns the disk path for a project using legacy storage + class LegacyProject + attr_accessor :project + + def initialize(project) + @project = project + end + + def disk_path + project.full_path + end + end + end + + # Concern used by Project and Namespace to determine the full route to the project + module Routable + extend ActiveSupport::Concern + + def full_path + route&.path || build_full_path + end + + def build_full_path + return path unless has_parent? + + raise OrphanedNamespaceError if parent.nil? + + parent.full_path + '/' + path + end + + def has_parent? + read_attribute(association(:parent).reflection.foreign_key) + end + end + + # Route model + class Route < ActiveRecord::Base + belongs_to :source, inverse_of: :route, polymorphic: true + end + + # Namespace model + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + self.inheritance_column = nil + + include Routable + + belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + + has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id + + has_many :projects, inverse_of: :parent + has_many :namespaces, inverse_of: :parent + end + + # ProjectRegistry model + class ProjectRepository < ActiveRecord::Base + self.table_name = 'project_repositories' + + belongs_to :project, inverse_of: :project_repository + end + + # Project model + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include Routable + + HASHED_STORAGE_FEATURES = { + repository: 1, + attachments: 2 + }.freeze + + scope :with_parent, -> { includes(:parent) } + + belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + + has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id + has_one :project_repository, inverse_of: :project + + delegate :disk_path, to: :storage + + class << self + def on_hashed_storage + where(Project.arel_table[:storage_version] + .gteq(HASHED_STORAGE_FEATURES[:repository])) + end + + def on_legacy_storage + where(Project.arel_table[:storage_version].eq(nil) + .or(Project.arel_table[:storage_version].eq(0))) + end + + def without_project_repository + joins(left_outer_join_project_repository) + .where(ProjectRepository.arel_table[:project_id].eq(nil)) + end + + def left_outer_join_project_repository + projects_table = Project.arel_table + repository_table = ProjectRepository.arel_table + + projects_table + .join(repository_table, Arel::Nodes::OuterJoin) + .on(projects_table[:id].eq(repository_table[:project_id])) + .join_sources + end + end + + def storage + @storage ||= + if hashed_storage? + Storage::HashedProject.new(self) + else + Storage::LegacyProject.new(self) + end + end + + def hashed_storage? + self.storage_version && + self.storage_version >= HASHED_STORAGE_FEATURES[:repository] + end + end + + def perform(start_id, stop_id) + Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) + end + + private + + def projects + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + def project_repositories(start_id, stop_id) + projects + .without_project_repository + .includes(:route, parent: [:route]).references(:routes) + .includes(:parent).references(:namespaces) + .where(id: start_id..stop_id) + .map { |project| build_attributes_for_project(project) } + .compact + end + + def build_attributes_for_project(project) + { + project_id: project.id, + shard_id: find_shard_id(project.repository_storage), + disk_path: project.disk_path + } + end + + def find_shard_id(repository_storage) + shard_finder.find_shard_id(repository_storage) + end + + def shard_finder + @shard_finder ||= ShardFinder.new + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb new file mode 100644 index 00000000000..d3f366f3480 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentRename < CleanupConcurrentSchemaChange + RESCHEDULE_DELAY = 10.minutes + + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_rename(table, old_column, new_column) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb new file mode 100644 index 00000000000..54f77f184d5 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for cleaning up concurrent schema changes. + class CleanupConcurrentSchemaChange + include Database::MigrationHelpers + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(table, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + self.class.name, + [table, old_column, new_column] + ) + else + cleanup_concurrent_schema_change(table, old_column, new_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb index de622f657b2..48411095dbb 100644 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -2,52 +2,12 @@ module Gitlab module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentTypeChange - include Database::MigrationHelpers - + # Background migration for cleaning up a concurrent column type changeb. + class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange RESCHEDULE_DELAY = 10.minutes - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(:issues, new_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - 'CleanupConcurrentTypeChange', - [table, old_column, new_column] - ) - else - cleanup_concurrent_column_type_change(table, old_column) - end - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_type_change(table, old_column) end end end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 1b4a9e8a194..ccd1f9b4dba 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb index c2bf42f846d..da8265a3a5f 100644 --- a/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb +++ b/lib/gitlab/background_migration/create_gpg_key_subkeys_from_gpg_keys.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation class Gitlab::BackgroundMigration::CreateGpgKeySubkeysFromGpgKeys diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb new file mode 100644 index 00000000000..664ead1af44 --- /dev/null +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class DeleteDiffFiles + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + belongs_to :merge_request + has_many :merge_request_diff_files + end + + class MergeRequestDiffFile < ActiveRecord::Base + self.table_name = 'merge_request_diff_files' + end + + DEAD_TUPLES_THRESHOLD = 50_000 + VACUUM_WAIT_TIME = 5.minutes + + def perform(ids) + @ids = ids + + # We should reschedule until deadtuples get in a desirable + # state (e.g. < 50_000). That may take more than one reschedule. + # + if should_wait_deadtuple_vacuum? + reschedule + return + end + + prune_diff_files + end + + def should_wait_deadtuple_vacuum? + return false unless Gitlab::Database.postgresql? + + diff_files_dead_tuples_count >= DEAD_TUPLES_THRESHOLD + end + + private + + def reschedule + BackgroundMigrationWorker.perform_in(VACUUM_WAIT_TIME, self.class.name.demodulize, [@ids]) + end + + def diffs_collection + MergeRequestDiff.where(id: @ids) + end + + def diff_files_dead_tuples_count + dead_tuple = + execute_statement("SELECT n_dead_tup FROM pg_stat_all_tables "\ + "WHERE relname = 'merge_request_diff_files'")[0] + + dead_tuple&.fetch('n_dead_tup', 0).to_i + end + + def prune_diff_files + removed = 0 + updated = 0 + + MergeRequestDiff.transaction do + updated = diffs_collection.update_all(state: 'without_files') + removed = MergeRequestDiffFile.where(merge_request_diff_id: @ids).delete_all + end + + log_info("Removed #{removed} merge_request_diff_files rows, "\ + "updated #{updated} merge_request_diffs rows") + end + + def execute_statement(sql) + ActiveRecord::Base.connection.execute(sql) + end + + def log_info(message) + Rails.logger.info("BackgroundMigration::DeleteDiffFiles - #{message}") + end + end + end +end diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index a357538a885..58df74cfa9b 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/digest_column.rb b/lib/gitlab/background_migration/digest_column.rb new file mode 100644 index 00000000000..22a3bb8f8f3 --- /dev/null +++ b/lib/gitlab/background_migration/digest_column.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class DigestColumn + class PersonalAccessToken < ActiveRecord::Base + self.table_name = 'personal_access_tokens' + end + + def perform(model, attribute_from, attribute_to, start_id, stop_id) + model = model.constantize if model.is_a?(String) + + model.transaction do + relation = model.where(id: start_id..stop_id).where.not(attribute_from => nil).lock + + relation.each do |instance| + instance.update_columns(attribute_to => Gitlab::CryptoHelper.sha256(instance.read_attribute(attribute_from)), + attribute_from => nil) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb new file mode 100644 index 00000000000..b9ad8267e37 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_columns.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # To avoid depending on a particular version of the model in app/, add a + # model to `lib/gitlab/background_migration/models/encrypt_columns` and use + # it in the migration that enqueues the jobs, so code can be shared. + # + # For this background migration to work, the table that is migrated _has_ to + # have an `id` column as the primary key. Additionally, the encrypted column + # should be managed by attr_encrypted, and map to an attribute with the same + # name as the unencrypted column (i.e., the unencrypted column should be + # shadowed), unless you want to define specific methods / accessors in the + # temporary model in `/models/encrypt_columns/your_model.rb`. + # + class EncryptColumns + def perform(model, attributes, from, to) + model = model.constantize if model.is_a?(String) + + # If sidekiq hasn't undergone a restart, its idea of what columns are + # present may be inaccurate, so ensure this is as fresh as possible + model.reset_column_information + model.define_attribute_methods + + attributes = expand_attributes(model, Array(attributes).map(&:to_sym)) + + model.transaction do + # Use SELECT ... FOR UPDATE to prevent the value being changed while + # we are encrypting it + relation = model.where(id: from..to).lock + + relation.each do |instance| + encrypt!(instance, attributes) + end + end + end + + def clear_migrated_values? + true + end + + private + + # Build a hash of { attribute => encrypted column name } + def expand_attributes(klass, attributes) + expanded = attributes.flat_map do |attribute| + attr_config = klass.encrypted_attributes[attribute] + crypt_column_name = attr_config&.fetch(:attribute) + + raise "Couldn't determine encrypted column for #{klass}##{attribute}" if + crypt_column_name.nil? + + raise "#{klass} source column: #{attribute} is missing" unless + klass.column_names.include?(attribute.to_s) + + # Running the migration without the destination column being present + # leads to data loss + raise "#{klass} destination column: #{crypt_column_name} is missing" unless + klass.column_names.include?(crypt_column_name.to_s) + + [attribute, crypt_column_name] + end + + Hash[*expanded] + end + + # Generate ciphertext for each column and update the database + def encrypt!(instance, attributes) + to_clear = attributes + .map { |plain, crypt| apply_attribute!(instance, plain, crypt) } + .compact + .flat_map { |plain| [plain, nil] } + + to_clear = Hash[*to_clear] + + if instance.changed? + instance.save! + + if clear_migrated_values? + instance.update_columns(to_clear) + end + end + end + + def apply_attribute!(instance, plain_column, crypt_column) + plaintext = instance[plain_column] + ciphertext = instance[crypt_column] + + # No need to do anything if the plaintext is nil, or an encrypted + # value already exists + return nil unless plaintext.present? && !ciphertext.present? + + # attr_encrypted will calculate and set the expected value for us + instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend + + plain_column + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb new file mode 100644 index 00000000000..91e559a8765 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_runners_tokens.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # EncryptColumn migrates data from an unencrypted column - `foo`, say - to + # an encrypted column - `encrypted_foo`, say. + # + # We only create a subclass here because we want to isolate this migration + # (migrating unencrypted runner registration tokens to encrypted columns) + # from other `EncryptColumns` migration. This class name is going to be + # serialized and stored in Redis and later picked by Sidekiq, so we need to + # create a separate class name in order to isolate these migration tasks. + # + # We can solve this differently, see tech debt issue: + # + # https://gitlab.com/gitlab-org/gitlab-ce/issues/54328 + # + class EncryptRunnersTokens < EncryptColumns + def perform(model, from, to) + resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}" + model = resource.constantize + attributes = model.encrypted_attributes.keys + + super(model, attributes, from, to) + end + + def clear_migrated_values? + false + end + end + end +end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb index 22b0ac71920..103bd98af14 100644 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb index d0816ae3ed5..77c1f1ffaf0 100644 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb index 94c65459a67..cba3e21cea6 100644 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ b/lib/gitlab/background_migration/fill_store_upload.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb new file mode 100644 index 00000000000..0a12401c35f --- /dev/null +++ b/lib/gitlab/background_migration/fix_cross_project_label_links.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class FixCrossProjectLabelLinks + GROUP_NESTED_LEVEL = 10.freeze + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + class Label < ActiveRecord::Base + self.inheritance_column = :_type_disabled + self.table_name = 'labels' + end + + class LabelLink < ActiveRecord::Base + self.table_name = 'label_links' + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + end + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + end + + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + self.table_name = 'namespaces' + + def self.groups_with_descendants_ids(start_id, stop_id) + # To isolate migration code, we avoid usage of + # Gitlab::GroupHierarchy#base_and_descendants which already + # does this job better + ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id) + group_ids = ids + + GROUP_NESTED_LEVEL.times do + ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id) + break if ids.empty? + + group_ids += ids + end + + group_ids.uniq + end + end + + def perform(start_id, stop_id) + group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id) + project_ids = Project.where(namespace_id: group_ids).select(:id) + + fix_issues(project_ids) + fix_merge_requests(project_ids) + end + + private + + # select IDs of issues which reference a label which is: + # a) a project label of a different project, or + # b) a group label of a different group than issue's project group + def fix_issues(project_ids) + issue_ids = Label + .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\' + INNER JOIN issues ON issues.id = label_links.target_id + INNER JOIN projects ON projects.id = issues.project_id') + .where('issues.project_id in (?)', project_ids) + .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\ + 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') + .select('distinct issues.id') + + Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) } + end + + # select IDs of MRs which reference a label which is: + # a) a project label of a different project, or + # b) a group label of a different group than MR's project group + def fix_merge_requests(project_ids) + mr_ids = Label + .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\' + INNER JOIN merge_requests ON merge_requests.id = label_links.target_id + INNER JOIN projects ON projects.id = merge_requests.target_project_id') + .where('merge_requests.target_project_id in (?)', project_ids) + .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\ + 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') + .select('distinct merge_requests.id') + + MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) } + end + + def check_resource_labels(resource, project_id) + local_labels = available_labels(project_id) + + # get all label links for the given resource (issue/MR) + # which reference a label not included in avaiable_labels + # (other than its project labels and labels of ancestor groups) + cross_labels = LabelLink + .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id') + .joins('INNER JOIN labels ON labels.id = label_links.label_id') + .where(target_type: resource.class.name.demodulize, target_id: resource.id) + .where('labels.id not in (?)', local_labels.select(:id)) + + cross_labels.each do |label| + matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color} + + next unless matching_label + + Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" + LabelLink.update(label.label_link_id, label_id: matching_label.id) + end + end + + # get all labels available for the project (including + # group labels of ancestor groups) + def available_labels(project_id) + @labels ||= {} + @labels[project_id] ||= Label + .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)", + project_group_ids(project_id), + project_id) + end + + def project_group_ids(project_id) + ids = [Project.find(project_id).namespace_id] + + GROUP_NESTED_LEVEL.times do + group = Namespace.find(ids.last) + break unless group.parent_id + + ids << group.parent_id + end + + ids + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb index 242e3143e71..268c6083d3c 100644 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ b/lib/gitlab/background_migration/migrate_build_stage.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/AbcSize # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 7088aa0860a..38fecac1bfe 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/migrate_legacy_artifacts.rb b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb new file mode 100644 index 00000000000..5cd638083b0 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_legacy_artifacts.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/ClassLength + +module Gitlab + module BackgroundMigration + ## + # The class to migrate job artifacts from `ci_builds` to `ci_job_artifacts` + class MigrateLegacyArtifacts + FILE_LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL + ARCHIVE_FILE_TYPE = 1 # equal to Ci::JobArtifact.file_types['archive'] + METADATA_FILE_TYPE = 2 # equal to Ci::JobArtifact.file_types['metadata'] + LEGACY_PATH_FILE_LOCATION = 1 # equal to Ci::JobArtifact.file_location['legacy_path'] + + def perform(start_id, stop_id) + ActiveRecord::Base.transaction do + insert_archives(start_id, stop_id) + insert_metadatas(start_id, stop_id) + delete_legacy_artifacts(start_id, stop_id) + end + end + + private + + def insert_archives(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_file, + artifacts_size, + COALESCE(artifacts_file_store, #{FILE_LOCAL_STORE}), + #{ARCHIVE_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{ARCHIVE_FILE_TYPE}) + SQL + end + + def insert_metadatas(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO + ci_job_artifacts ( + project_id, + job_id, + expire_at, + file_location, + created_at, + updated_at, + file, + size, + file_store, + file_type + ) + SELECT + project_id, + id, + artifacts_expire_at, + #{LEGACY_PATH_FILE_LOCATION}, + created_at, + created_at, + artifacts_metadata, + NULL, + COALESCE(artifacts_metadata_store, #{FILE_LOCAL_STORE}), + #{METADATA_FILE_TYPE} + FROM + ci_builds + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + AND artifacts_metadata <> '' + AND NOT EXISTS ( + SELECT + 1 + FROM + ci_job_artifacts + WHERE + ci_builds.id = ci_job_artifacts.job_id + AND ci_job_artifacts.file_type = #{METADATA_FILE_TYPE}) + SQL + end + + def delete_legacy_artifacts(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + ci_builds + SET + artifacts_file = NULL, + artifacts_file_store = NULL, + artifacts_size = NULL, + artifacts_metadata = NULL, + artifacts_metadata_store = NULL + WHERE + id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} + AND artifacts_file <> '' + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb index 0e5c7f092f2..6a29a632577 100644 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ b/lib/gitlab/background_migration/migrate_stage_status.rb @@ -16,10 +16,10 @@ module Gitlab scope :running, -> { where(status: 'running') } scope :pending, -> { where(status: 'pending') } scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } + scope :failed, -> { where(status: 'failed') } + scope :canceled, -> { where(status: 'canceled') } + scope :skipped, -> { where(status: 'skipped') } + scope :manual, -> { where(status: 'manual') } scope :failed_but_allowed, -> do where(allow_failure: true, status: [:failed, :canceled]) diff --git a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb index 7f243073fd0..ef50fe4adb1 100644 --- a/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb +++ b/lib/gitlab/background_migration/migrate_system_uploads_to_new_folder.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb new file mode 100644 index 00000000000..41f18979d76 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `namespaces` table. + # + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb new file mode 100644 index 00000000000..bfeae14584d --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/project.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `projects` table. + # + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + def runners_token=(value) + self.runners_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { runners_token: { attribute: :runners_token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb new file mode 100644 index 00000000000..14ddce4b147 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` column in `ci_runners` table. + # + class Runner < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_runners' + self.inheritance_column = :_type_disabled + + def token=(value) + self.token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { token: { attribute: :token_encrypted } } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb new file mode 100644 index 00000000000..08ae35c0671 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `runners_token` column in `application_settings` table. + # + class Settings < ActiveRecord::Base + include ::EachBatch + include ::CacheableAttributes + + self.table_name = 'application_settings' + self.inheritance_column = :_type_disabled + + after_commit do + ::ApplicationSetting.expire + end + + def runners_registration_token=(value) + self.runners_registration_token_encrypted = + ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) + end + + def self.encrypted_attributes + { + runners_registration_token: { + attribute: :runners_registration_token_encrypted + } + } + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb new file mode 100644 index 00000000000..34e72fd9f34 --- /dev/null +++ b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Models + module EncryptColumns + # This model is shared between synchronous and background migrations to + # encrypt the `token` and `url` columns + class WebHook < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'web_hooks' + self.inheritance_column = :_type_disabled + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: ::Settings.attr_encrypted_db_key_base_32 + + attr_encrypted :url, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: ::Settings.attr_encrypted_db_key_base_32 + end + end + end + end +end diff --git a/lib/gitlab/background_migration/move_personal_snippet_files.rb b/lib/gitlab/background_migration/move_personal_snippet_files.rb index a4ef51fd0e8..5b2b2af718a 100644 --- a/lib/gitlab/background_migration/move_personal_snippet_files.rb +++ b/lib/gitlab/background_migration/move_personal_snippet_files.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb index d9d3d2e667b..698f5e46c0c 100644 --- a/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb +++ b/lib/gitlab/background_migration/normalize_ldap_extern_uids_range.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/LineLength # rubocop:disable Metrics/ClassLength # rubocop:disable Metrics/BlockLength # rubocop:disable Style/Documentation diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb new file mode 100644 index 00000000000..35bfc381180 --- /dev/null +++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +# +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateClusterKubernetesNamespaceTable + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + module Migratable + class KubernetesNamespace < ActiveRecord::Base + self.table_name = 'clusters_kubernetes_namespaces' + end + + class ClusterProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'cluster_projects' + + belongs_to :project + + def self.with_no_kubernetes_namespace + where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) + end + + def namespace + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') + end + + def service_account + "#{namespace}-service-account" + end + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + end + + def perform + cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| + sql_values = sql_values_for(cluster_projects_batch) + + insert_into_cluster_kubernetes_namespace(sql_values) + end + end + + private + + def cluster_projects_with_no_kubernetes_namespace + Migratable::ClusterProject.with_no_kubernetes_namespace + end + + def sql_values_for(cluster_projects) + cluster_projects.map do |cluster_project| + values_for_cluster_project(cluster_project) + end + end + + def values_for_cluster_project(cluster_project) + { + cluster_project_id: cluster_project.id, + cluster_id: cluster_project.cluster_id, + project_id: cluster_project.project_id, + namespace: cluster_project.namespace, + service_account_name: cluster_project.service_account, + created_at: 'NOW()', + updated_at: 'NOW()' + } + end + + def insert_into_cluster_kubernetes_namespace(rows) + Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, + rows, + disable_quote: [:created_at, :updated_at]) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb new file mode 100644 index 00000000000..036fe641757 --- /dev/null +++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateExternalPipelineSource + module Migratable + class Pipeline < ActiveRecord::Base + self.table_name = 'ci_pipelines' + + def self.sources + { + unknown: nil, + push: 1, + web: 2, + trigger: 3, + schedule: 4, + api: 5, + external: 6 + } + end + end + + class CommitStatus < ActiveRecord::Base + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') } + scope :of_type, -> (type) { where('type=?', type) } + end + end + + def perform(start_id, stop_id) + external_pipelines(start_id, stop_id) + .update_all(source: Migratable::Pipeline.sources[:external]) + end + + private + + def external_pipelines(start_id, stop_id) + Migratable::Pipeline.where(id: (start_id..stop_id)) + .where( + 'EXISTS (?) AND NOT EXISTS (?)', + Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1), + Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1) + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_fork_networks_range.rb b/lib/gitlab/background_migration/populate_fork_networks_range.rb index a976cb4c243..aa4f130538c 100644 --- a/lib/gitlab/background_migration/populate_fork_networks_range.rb +++ b/lib/gitlab/background_migration/populate_fork_networks_range.rb @@ -19,7 +19,7 @@ module Gitlab create_fork_networks_for_missing_projects(start_id, end_id) create_fork_networks_memberships_for_root_projects(start_id, end_id) - delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength + delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY BackgroundMigrationWorker.perform_in( delay, "CreateForkNetworkMembershipsRange", [start_id, end_id] ) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb index 8a901a9bf39..d89ce358bb9 100644 --- a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -1,7 +1,4 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength -# rubocop:disable Metrics/MethodLength -# rubocop:disable Metrics/ClassLength # rubocop:disable Style/Documentation module Gitlab diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb new file mode 100644 index 00000000000..37592d67dd9 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_improved.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestMetricsWithEventsDataImproved + CLOSED_EVENT_ACTION = 3 + MERGED_EVENT_ACTION = 7 + + def perform(min_merge_request_id, max_merge_request_id) + insert_metrics_for_range(min_merge_request_id, max_merge_request_id) + update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) + end + + # Inserts merge_request_metrics records for merge_requests without it for + # a given merge request batch. + def insert_metrics_for_range(min, max) + metrics_not_exists_clause = + <<-SQL.strip_heredoc + NOT EXISTS (SELECT 1 FROM merge_request_metrics + WHERE merge_request_metrics.merge_request_id = merge_requests.id) + SQL + + MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| + select_sql = batch.select(:id, :created_at, :updated_at).to_sql + + execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") + end + end + + def update_metrics_with_events_data(min, max) + if Gitlab::Database.postgresql? + psql_update_metrics_with_events_data(min, max) + else + mysql_update_metrics_with_events_data(min, max) + end + end + + def psql_update_metrics_with_events_data(min, max) + update_sql = <<-SQL.strip_heredoc + UPDATE merge_request_metrics + SET (latest_closed_at, + latest_closed_by_id) = + ( SELECT updated_at, + author_id + FROM events + WHERE target_id = merge_request_id + AND target_type = 'MergeRequest' + AND action = #{CLOSED_EVENT_ACTION} + ORDER BY id DESC + LIMIT 1 ), + merged_by_id = + ( SELECT author_id + FROM events + WHERE target_id = merge_request_id + AND target_type = 'MergeRequest' + AND action = #{MERGED_EVENT_ACTION} + ORDER BY id DESC + LIMIT 1 ) + WHERE merge_request_id BETWEEN #{min} AND #{max} + SQL + + execute(update_sql) + end + + def mysql_update_metrics_with_events_data(min, max) + closed_updated_at_subquery = mysql_events_select(:updated_at, CLOSED_EVENT_ACTION) + closed_author_id_subquery = mysql_events_select(:author_id, CLOSED_EVENT_ACTION) + merged_author_id_subquery = mysql_events_select(:author_id, MERGED_EVENT_ACTION) + + update_sql = <<-SQL.strip_heredoc + UPDATE merge_request_metrics + SET latest_closed_at = (#{closed_updated_at_subquery}), + latest_closed_by_id = (#{closed_author_id_subquery}), + merged_by_id = (#{merged_author_id_subquery}) + WHERE merge_request_id BETWEEN #{min} AND #{max} + SQL + + execute(update_sql) + end + + def mysql_events_select(column, action) + <<-SQL.strip_heredoc + SELECT #{column} FROM events + WHERE target_id = merge_request_id + AND target_type = 'MergeRequest' + AND action = #{action} + ORDER BY id DESC + LIMIT 1 + SQL + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9232f20a063..a19dc9747fb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,7 +4,7 @@ module Gitlab module BackgroundMigration # This class processes a batch of rows in `untracked_files_for_uploads` by # adding each file to the `uploads` table if it does not exist. - class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + class PopulateUntrackedUploads def perform(start_id, end_id) return unless migrate? diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a2c5acbde71..4a9a62aaeb5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -4,7 +4,7 @@ module Gitlab module PopulateUntrackedUploadsDependencies # This class is responsible for producing the attributes necessary to # track an uploaded file in the `uploads` table. - class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename @@ -134,7 +134,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 914a9e48a2f..81ca2b0a9b7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -54,7 +54,8 @@ module Gitlab def ensure_temporary_tracking_table_exists table_name = :untracked_files_for_uploads - unless UntrackedFile.connection.table_exists?(table_name) + + unless ActiveRecord::Base.connection.data_source_exists?(table_name) UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true @@ -143,7 +144,7 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb new file mode 100644 index 00000000000..92256e59a6c --- /dev/null +++ b/lib/gitlab/background_migration/redact_links.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +require_relative 'redact_links/redactable' + +module Gitlab + module BackgroundMigration + class RedactLinks + class Note < ActiveRecord::Base + include EachBatch + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable + + self.table_name = 'notes' + self.inheritance_column = :_type_disabled + end + + class Issue < ActiveRecord::Base + include EachBatch + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable + + self.table_name = 'issues' + self.inheritance_column = :_type_disabled + end + + class MergeRequest < ActiveRecord::Base + include EachBatch + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable + + self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled + end + + class Snippet < ActiveRecord::Base + include EachBatch + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable + + self.table_name = 'snippets' + self.inheritance_column = :_type_disabled + end + + def perform(model_name, field, start_id, stop_id) + link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%" + model = "Gitlab::BackgroundMigration::RedactLinks::#{model_name}".constantize + + model.where("#{field} like ?", link_pattern).where(id: start_id..stop_id).each do |resource| + resource.redact_field!(field) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/redact_links/redactable.rb b/lib/gitlab/background_migration/redact_links/redactable.rb new file mode 100644 index 00000000000..baab34221f1 --- /dev/null +++ b/lib/gitlab/background_migration/redact_links/redactable.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RedactLinks + module Redactable + extend ActiveSupport::Concern + + def redact_field!(field) + self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') + + if self.changed? + self.update_columns(field => self[field], + "#{field}_html" => nil) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb new file mode 100644 index 00000000000..47579d46c1b --- /dev/null +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +# rubocop:disable Metrics/ClassLength + +module Gitlab + module BackgroundMigration + class RemoveRestrictedTodos + PRIVATE_FEATURE = 10 + PRIVATE_PROJECT = 0 + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + class ProjectAuthorization < ActiveRecord::Base + self.table_name = 'project_authorizations' + end + + class ProjectFeature < ActiveRecord::Base + self.table_name = 'project_features' + end + + class Todo < ActiveRecord::Base + include EachBatch + + self.table_name = 'todos' + end + + class Issue < ActiveRecord::Base + include EachBatch + + self.table_name = 'issues' + end + + def perform(start_id, stop_id) + projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)') + .where(id: start_id..stop_id) + + projects.each do |project| + remove_confidential_issue_todos(project.id) + + if project.visibility_level == PRIVATE_PROJECT + remove_non_members_todos(project.id) + else + remove_restricted_features_todos(project.id) + end + end + end + + private + + def remove_non_members_todos(project_id) + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id) + else + unauthorized_project_todos(project_id) + .each_batch(of: 5000) do |batch| + batch.delete_all + end + end + end + + def remove_confidential_issue_todos(project_id) + # min access level to access a confidential issue is reporter + min_reporters = authorized_users(project_id) + .select(:user_id) + .where('access_level >= ?', 20) + + confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) + confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| + batch.each do |issue| + assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) + + todos = Todo.where(target_type: 'Issue', target_id: issue.id) + .where('user_id NOT IN (?)', min_reporters) + .where('user_id NOT IN (?)', assigned_users) + todos = todos.where('user_id != ?', issue.author_id) if issue.author_id + + todos.delete_all + end + end + end + + def remove_restricted_features_todos(project_id) + ProjectFeature.where(project_id: project_id).each do |project_features| + target_types = [] + target_types << 'Issue' if private?(project_features.issues_access_level) + target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level) + target_types << 'Commit' if private?(project_features.repository_access_level) + + next if target_types.empty? + + if Gitlab::Database.postgresql? + batch_remove_todos_cte(project_id, target_types) + else + unauthorized_project_todos(project_id) + .where(target_type: target_types) + .delete_all + end + end + end + + def private?(feature_level) + feature_level == PRIVATE_FEATURE + end + + def authorized_users(project_id) + ProjectAuthorization.select(:user_id).where(project_id: project_id) + end + + def unauthorized_project_todos(project_id) + Todo.where(project_id: project_id) + .where('user_id NOT IN (?)', authorized_users(project_id)) + end + + def batch_remove_todos_cte(project_id, target_types = nil) + loop do + count = remove_todos_cte(project_id, target_types) + + break if count == 0 + end + end + + def remove_todos_cte(project_id, target_types = nil) + sql = [] + sql << with_all_todos_sql(project_id, target_types) + sql << as_deleted_sql + sql << "SELECT count(*) FROM deleted" + + result = Todo.connection.exec_query(sql.join(' ')) + result.rows[0][0].to_i + end + + def with_all_todos_sql(project_id, target_types = nil) + if target_types + table = Arel::Table.new(:todos) + in_target = table[:target_type].in(target_types) + target_types_sql = " AND #{in_target.to_sql}" + end + + <<-SQL + WITH all_todos AS ( + SELECT id + FROM "todos" + WHERE "todos"."project_id" = #{project_id} + AND (user_id NOT IN ( + SELECT "project_authorizations"."user_id" + FROM "project_authorizations" + WHERE "project_authorizations"."project_id" = #{project_id}) + #{target_types_sql} + ) + ), + SQL + end + + def as_deleted_sql + <<-SQL + deleted AS ( + DELETE FROM todos + WHERE id IN ( + SELECT id + FROM all_todos + LIMIT 5000 + ) + RETURNING id + ) + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/schedule_diff_files_deletion.rb b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb new file mode 100644 index 00000000000..609cf19187c --- /dev/null +++ b/lib/gitlab/background_migration/schedule_diff_files_deletion.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ScheduleDiffFilesDeletion + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + belongs_to :merge_request + + include EachBatch + end + + DIFF_BATCH_SIZE = 5_000 + INTERVAL = 5.minutes + MIGRATION = 'DeleteDiffFiles' + + def perform + diffs = MergeRequestDiff + .from("(#{diffs_collection.to_sql}) merge_request_diffs") + .where('merge_request_diffs.id != merge_request_diffs.latest_merge_request_diff_id') + .select(:id) + + diffs.each_batch(of: DIFF_BATCH_SIZE) do |relation, index| + ids = relation.pluck(:id) + + BackgroundMigrationWorker.perform_in(index * INTERVAL, MIGRATION, [ids]) + end + end + + private + + def diffs_collection + MergeRequestDiff + .joins(:merge_request) + .where("merge_requests.state = 'merged'") + .where('merge_requests.latest_merge_request_diff_id IS NOT NULL') + .where("merge_request_diffs.state NOT IN ('without_files', 'empty')") + .select('merge_requests.latest_merge_request_diff_id, merge_request_diffs.id') + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb index e5e8837221e..bc434b0cb64 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures services which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures services which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnServices class Service < ActiveRecord::Base self.table_name = 'services' diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb index 171c8ef21b7..28d8d2c640b 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures hooks which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures hooks which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnWebhooks class WebHook < ActiveRecord::Base self.table_name = 'web_hooks' diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb index 909fa24fa90..fb55b9e2f1f 100644 --- a/lib/gitlab/badge/base.rb +++ b/lib/gitlab/badge/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge class Base diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb index e898f5d790e..9181ba2d4b0 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Coverage diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 778d78185ff..7f7cc62c8ef 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Coverage @@ -12,7 +14,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.pipelines.latest_successful_for(@ref) + @pipeline = @project.ci_pipelines.latest_successful_for(@ref) end def entity @@ -36,6 +38,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def raw_coverage return unless @pipeline @@ -47,6 +50,7 @@ module Gitlab .try(:coverage) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index afbf9dd17e3..817dc28f84a 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Coverage diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 8ad6f3cb986..b9ae68134b0 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge ## diff --git a/lib/gitlab/badge/pipeline/metadata.rb b/lib/gitlab/badge/pipeline/metadata.rb index db1e9f8cfb8..d4d789558c9 100644 --- a/lib/gitlab/badge/pipeline/metadata.rb +++ b/lib/gitlab/badge/pipeline/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Pipeline diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index 5fee7a93475..a403d839517 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Pipeline @@ -18,11 +20,13 @@ module Gitlab 'pipeline' end + # rubocop: disable CodeReuse/ActiveRecord def status - @project.pipelines + @project.ci_pipelines .where(sha: @sha) .latest_status(@ref) || 'unknown' end + # rubocop: enable CodeReuse/ActiveRecord def metadata @metadata ||= Pipeline::Metadata.new(self) diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb index e09db32262d..64c3dfcd10b 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge module Pipeline diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb index bfeb0052642..ed2ec50b197 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/badge/template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Badge ## diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 4ca5a78e068..3cd327f5109 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -1,10 +1,15 @@ +# frozen_string_literal: true + module Gitlab module BareRepositoryImport class Importer NoAdminError = Class.new(StandardError) def self.execute(import_path) - import_path << '/' unless import_path.ends_with?('/') + unless import_path.ends_with?('/') + import_path = "#{import_path}/" + end + repos_to_import = Dir.glob(import_path + '**/*.git') unless user = User.admins.order_id_asc.first @@ -26,6 +31,12 @@ module Gitlab end end + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + attr_reader :user, :project_name, :bare_repo delegate :log, to: :class @@ -59,11 +70,10 @@ module Gitlab import_type: 'bare_repository', namespace_id: group&.id).execute - if project.persisted? && mv_repo(project) + if project.persisted? && mv_repositories(project) log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else @@ -74,12 +84,11 @@ module Gitlab project end - def mv_repo(project) - storage_path = storage_path_for_shard(project.repository_storage) - FileUtils.mv(repo_path, project.repository.path_to_repo) + def mv_repositories(project) + mv_repo(bare_repo.repo_path, project.repository) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) + mv_repo(bare_repo.wiki_path, project.wiki.repository) end true @@ -89,6 +98,11 @@ module Gitlab false end + def mv_repo(path, repository) + repository.create_from_bundle(bundle(path)) + FileUtils.rm_rf(path) + end + def storage_path_for_shard(shard) Gitlab.config.repositories.storages[shard].legacy_disk_path end @@ -101,10 +115,17 @@ module Gitlab Groups::NestedCreateService.new(user, group_path: group_path).execute end - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - def self.log(message) - puts message # rubocop:disable Rails/Output + def bundle(repo_path) + # TODO: we could save some time and disk space by using + # `git bundle create - --all` and streaming the bundle directly to + # Gitaly, rather than writing it on disk first + bundle_path = "#{repo_path}.bundle" + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) + output, status = Gitlab::Popen.popen(cmd) + + raise output unless status.zero? + + bundle_path end end end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fe267248275..b903c581aac 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,5 +1,5 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953 -# +# frozen_string_literal: true + module Gitlab module BareRepositoryImport class Repository @@ -8,9 +8,12 @@ module Gitlab attr_reader :group_path, :project_name, :repo_path def initialize(root_path, repo_path) + unless root_path.ends_with?('/') + root_path = "#{root_path}/" + end + @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') full_path = if hashed? && !wiki? diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index e4227af25d2..b78993aba30 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f3999e690fa..eaead41a720 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module BitbucketImport class Importer @@ -33,7 +35,7 @@ module Gitlab def handle_errors return unless errors.any? - project.update_column(:import_error, { + project.import_state.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) @@ -43,6 +45,7 @@ module Gitlab find_user_id(username) || project.creator_id end + # rubocop: disable CodeReuse/ActiveRecord def find_user_id(username) return nil unless username @@ -53,6 +56,7 @@ module Gitlab .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord def repo @repo ||= client.repo(project.import_source) @@ -68,6 +72,7 @@ module Gitlab errors << { type: :wiki, errors: e.message } end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues_enabled? @@ -101,6 +106,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_comments(issue, gitlab_issue) client.issue_comments(repo, issue.iid).each do |comment| @@ -188,7 +194,8 @@ module Gitlab end def import_inline_comments(inline_comments, pull_request, merge_request) - line_code_map = {} + position_map = {} + discussion_map = {} children, parents = inline_comments.partition(&:has_parent?) @@ -196,22 +203,28 @@ module Gitlab # relationships. We assume that the child can appear in any order in # the JSON. parents.each do |comment| - line_code_map[comment.iid] = generate_line_code(comment) + position_map[comment.iid] = build_position(merge_request, comment) end children.each do |comment| - line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + position_map[comment.iid] = position_map.fetch(comment.parent_id, nil) end inline_comments.each do |comment| begin attributes = pull_request_comment_attributes(comment) + attributes[:discussion_id] = discussion_map[comment.parent_id] if comment.has_parent? + attributes.merge!( - position: build_position(merge_request, comment), - line_code: line_code_map.fetch(comment.iid), + position: position_map[comment.iid], type: 'DiffNote') - merge_request.notes.create!(attributes) + note = merge_request.notes.create!(attributes) + + # We can't store a discussion ID until a note is created, so if + # replies are created before the parent the discussion ID won't be + # linked properly. + discussion_map[comment.iid] = note.discussion_id rescue StandardError => e errors << { type: :pull_request, iid: comment.iid, errors: e.message } end @@ -240,10 +253,6 @@ module Gitlab end end - def generate_line_code(pr_comment) - Gitlab::Git.diff_line_code(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) - end - def pull_request_comment_attributes(comment) { project: project, diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index d94f70fd1fb..11070a68e02 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module BitbucketImport class ProjectCreator diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb new file mode 100644 index 00000000000..dbbedd5dcbe --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -0,0 +1,386 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class Importer + attr_reader :recover_missing_commits + attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + attr_accessor :logger + + REMOTE_NAME = 'bitbucket_server'.freeze + BATCH_SIZE = 100 + + TempBranch = Struct.new(:name, :sha) + + def self.imports_repository? + true + end + + def self.refmap + [:heads, :tags, '+refs/pull-requests/*/to:refs/merge-requests/*/head'] + end + + # Unlike GitHub, you can't grab the commit SHAs for pull requests that + # have been closed but not merged even though Bitbucket has these + # commits internally. We can recover these pull requests by creating a + # branch with the Bitbucket REST API, but by default we turn this + # behavior off. + def initialize(project, recover_missing_commits: false) + @project = project + @recover_missing_commits = recover_missing_commits + @project_key = project.import_data.data['project_key'] + @repository_slug = project.import_data.data['repo_slug'] + @client = BitbucketServer::Client.new(project.import_data.credentials) + @formatter = Gitlab::ImportFormatter.new + @errors = [] + @users = {} + @temp_branches = [] + @logger = Gitlab::Import::Logger.build + end + + def execute + import_repository + import_pull_requests + delete_temp_branches + handle_errors + + log_info(stage: "complete") + + true + end + + private + + def handle_errors + return unless errors.any? + + project.import_state.update_column(:last_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(email) + find_user_id(email) || project.creator_id + end + + def find_user_id(email) + return nil unless email + + return users[email] if users.key?(email) + + user = User.find_by_any_email(email, confirmed: true) + users[email] = user&.id + + user&.id + end + + def repo + @repo ||= client.repo(project_key, repository_slug) + end + + def sha_exists?(sha) + project.repository.commit(sha) + end + + def temp_branch_name(pull_request, suffix) + "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}" + end + + # This method restores required SHAs that GitLab needs to create diffs + # into branch names as the following: + # + # gitlab/import/pull-request/N/{to,from} + def restore_branches(pull_requests) + shas_to_restore = [] + + pull_requests.each do |pull_request| + shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from), + pull_request.source_branch_sha) + shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to), + pull_request.target_branch_sha) + end + + # Create the branches on the Bitbucket Server first + created_branches = restore_branch_shas(shas_to_restore) + + @temp_branches += created_branches + # Now sync the repository so we get the new branches + import_repository unless created_branches.empty? + end + + def restore_branch_shas(shas_to_restore) + shas_to_restore.each_with_object([]) do |temp_branch, branches_created| + branch_name = temp_branch.name + sha = temp_branch.sha + + next if sha_exists?(sha) + + begin + client.create_branch(project_key, repository_slug, branch_name, sha) + branches_created << temp_branch + rescue BitbucketServer::Connection::ConnectionError => e + log_warn(message: "Unable to recreate branch", sha: sha, error: e.message) + end + end + end + + def import_repository + log_info(stage: 'import_repository', message: 'starting import') + + project.ensure_repository + project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) + + log_info(stage: 'import_repository', message: 'finished import') + rescue Gitlab::Shell::Error => e + log_error(stage: 'import_repository', message: 'failed import', error: e.message) + + # Expire cache to prevent scenarios such as: + # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true + # 2. Retried import, repo is broken or not imported but +exists?+ still returns true + project.repository.expire_content_cache if project.repository_exists? + + raise + end + + # Bitbucket Server keeps tracks of references for open pull requests in + # refs/heads/pull-requests, but closed and merged requests get moved + # into hidden internal refs under stash-refs/pull-requests. Unless the + # SHAs involved are at the tip of a branch or tag, there is no way to + # retrieve the server for those commits. + # + # To avoid losing history, we use the Bitbucket API to re-create the branch + # on the remote server. Then we have to issue a `git fetch` to download these + # branches. + def import_pull_requests + pull_requests = client.pull_requests(project_key, repository_slug).to_a + + # Creating branches on the server and fetching the newly-created branches + # may take a number of network round-trips. Do this in batches so that we can + # avoid doing a git fetch for every new branch. + pull_requests.each_slice(BATCH_SIZE) do |batch| + restore_branches(batch) if recover_missing_commits + + batch.each do |pull_request| + begin + import_bitbucket_pull_request(pull_request) + rescue StandardError => e + backtrace = Gitlab::Profiler.clean_backtrace(e.backtrace) + log_error(stage: 'import_pull_requests', iid: pull_request.iid, error: e.message, backtrace: backtrace) + + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } + end + end + end + end + + def delete_temp_branches + @temp_branches.each do |branch| + begin + client.delete_branch(project_key, repository_slug, branch.name, branch.sha) + project.repository.delete_branch(branch.name) + rescue BitbucketServer::Connection::ConnectionError => e + log_error(stage: 'delete_temp_branches', branch: branch.name, error: e.message) + @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } + end + end + end + + def import_bitbucket_pull_request(pull_request) + log_info(stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) + + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author_email) + description += pull_request.description if pull_request.description + author_id = gitlab_user_id(pull_request.author_email) + + attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project_id: project.id, + source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name), + source_branch_sha: pull_request.source_branch_sha, + target_project_id: project.id, + target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: author_id, + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + creator = Gitlab::Import::MergeRequestCreator.new(project) + merge_request = creator.execute(attributes) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + + log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) + end + + def import_pull_request_comments(pull_request, merge_request) + log_info(stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) + + comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) + + merge_event = other_activities.find(&:merge_event?) + import_merge_event(merge_request, merge_event) if merge_event + + inline_comments, pr_comments = comments.partition(&:inline_comment?) + + import_inline_comments(inline_comments.map(&:comment), merge_request) + import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + + log_info(stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, + merge_event_found: merge_event.present?, + inline_comments_count: inline_comments.count, + standalone_pr_comments: pr_comments.count) + end + + # rubocop: disable CodeReuse/ActiveRecord + def import_merge_event(merge_request, merge_event) + log_info(stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) + + committer = merge_event.committer_email + + user_id = gitlab_user_id(committer) + timestamp = merge_event.merge_timestamp + merge_request.update({ merge_commit_sha: merge_event.merge_commit }) + metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) + metric.update(merged_by_id: user_id, merged_at: timestamp) + + log_info(stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) + end + # rubocop: enable CodeReuse/ActiveRecord + + def import_inline_comments(inline_comments, merge_request) + log_info(stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) + + inline_comments.each do |comment| + position = build_position(merge_request, comment) + parent = create_diff_note(merge_request, comment, position) + + next unless parent&.persisted? + + discussion_id = parent.discussion_id + + comment.comments.each do |reply| + create_diff_note(merge_request, reply, position, discussion_id) + end + end + + log_info(stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) + end + + def create_diff_note(merge_request, comment, position, discussion_id = nil) + attributes = pull_request_comment_attributes(comment) + attributes.merge!(position: position, type: 'DiffNote') + attributes[:discussion_id] = discussion_id if discussion_id + + note = merge_request.notes.build(attributes) + + if note.valid? + note.save + return note + end + + log_info(stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + + # Bitbucket Server supports the ability to comment on any line, not just the + # line in the diff. If we can't add the note as a DiffNote, fallback to creating + # a regular note. + create_fallback_diff_note(merge_request, comment, position) + rescue StandardError => e + log_error(stage: 'create_diff_note', comment_id: comment.id, error: e.message) + errors << { type: :pull_request, id: comment.id, errors: e.message } + nil + end + + def create_fallback_diff_note(merge_request, comment, position) + attributes = pull_request_comment_attributes(comment) + note = "*Comment on" + + note += " #{position.old_path}:#{position.old_line} -->" if position.old_line + note += " #{position.new_path}:#{position.new_line}" if position.new_line + note += "*\n\n#{comment.note}" + + attributes[:note] = note + merge_request.notes.create!(attributes) + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + + comment.comments.each do |replies| + merge_request.notes.create!(pull_request_comment_attributes(replies)) + end + rescue StandardError => e + log_error(stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message) + errors << { type: :pull_request, comment_id: comment.id, errors: e.message } + end + end + end + + def pull_request_comment_attributes(comment) + author = find_user_id(comment.author_email) + note = '' + + unless author + author = project.creator_id + note = "*By #{comment.author_username} (#{comment.author_email})*\n\n" + end + + note += + # Provide some context for replying + if comment.parent_comment + "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}" + else + comment.note + end + + { + project: project, + note: note, + author_id: author, + created_at: comment.created_at, + updated_at: comment.updated_at + } + end + + def log_info(details) + logger.info(log_base_data.merge(details)) + end + + def log_error(details) + logger.error(log_base_data.merge(details)) + end + + def log_warn(details) + logger.warn(log_base_data.merge(details)) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb new file mode 100644 index 00000000000..48ca4951957 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/project_creator.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + class ProjectCreator + attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data + + def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data) + @project_key = project_key + @repo_slug = repo_slug + @repo = repo + @name = name + @namespace = namespace + @current_user = current_user + @session_data = session_data + end + + def execute + ::Projects::CreateService.new( + current_user, + name: name, + path: name, + description: repo.description, + namespace_id: namespace.id, + visibility_level: repo.visibility_level, + import_type: 'bitbucket_server', + import_source: repo.browse_url, + import_url: repo.clone_url, + import_data: { + credentials: session_data, + data: { project_key: project_key, repo_slug: repo_slug } + }, + skip_wiki: true + ).execute + end + end + end +end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 169aac79854..f1a653a9d95 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class Blame attr_accessor :blob, :commit @@ -41,8 +43,7 @@ module Gitlab def highlighted_lines @blob.load_all_data! - @highlighted_lines ||= - Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines + @highlighted_lines ||= @blob.present.highlight.lines end def project diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb new file mode 100644 index 00000000000..d3e15a79a8b --- /dev/null +++ b/lib/gitlab/blob_helper.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +# This has been extracted from https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb +module Gitlab + module BlobHelper + def extname + File.extname(name.to_s) + end + + def known_extension? + LanguageData.extensions.include?(extname) + end + + def viewable? + !large? && text_in_repo? + end + + MEGABYTE = 1024 * 1024 + + def large? + size.to_i > MEGABYTE + end + + def binary_in_repo? + # Large blobs aren't even loaded into memory + if data.nil? + true + + # Treat blank files as text + elsif data == "" + false + + # Charlock doesn't know what to think + elsif encoding.nil? + true + + # If Charlock says its binary + else + detect_encoding[:type] == :binary + end + end + + def text_in_repo? + !binary_in_repo? + end + + def image? + ['.png', '.jpg', '.jpeg', '.gif'].include?(extname.downcase) + end + + # Internal: Lookup mime type for extension. + # + # Returns a MIME::Type + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def _mime_type + if defined? @_mime_type + @_mime_type + else + guesses = ::MIME::Types.type_for(extname.to_s) + + # Prefer text mime types over binary + @_mime_type = guesses.detect { |type| type.ascii? } || guesses.first + end + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + + # Public: Get the actual blob mime type + # + # Examples + # + # # => 'text/plain' + # # => 'text/html' + # + # Returns a mime type String. + def mime_type + _mime_type ? _mime_type.to_s : 'text/plain' + end + + def binary_mime_type? + _mime_type ? _mime_type.binary? : false + end + + def lines + @lines ||= + if viewable? && data + # `data` is usually encoded as ASCII-8BIT even when the content has + # been detected as a different encoding. However, we are not allowed + # to change the encoding of `data` because we've made the implicit + # guarantee that each entry in `lines` is encoded the same way as + # `data`. + # + # Instead, we re-encode each possible newline sequence as the + # detected encoding, then force them back to the encoding of `data` + # (usually a binary encoding like ASCII-8BIT). This means that the + # byte sequence will match how newlines are likely encoded in the + # file, but we don't have to change the encoding of `data` as far as + # Ruby is concerned. This allows us to correctly parse out each line + # without changing the encoding of `data`, and + # also--importantly--without having to duplicate many (potentially + # large) strings. + begin + data.split(encoded_newlines_re, -1) + rescue Encoding::ConverterNotFoundError + # The data is not splittable in the detected encoding. Assume it's + # one big line. + [data] + end + else + [] + end + end + + def content_type + # rubocop:disable Style/MultilineTernaryOperator + # rubocop:disable Style/NestedTernaryOperator + @content_type ||= binary_mime_type? || binary_in_repo? ? mime_type : + (encoding ? "text/plain; charset=#{encoding.downcase}" : "text/plain") + # rubocop:enable Style/NestedTernaryOperator + # rubocop:enable Style/MultilineTernaryOperator + end + + def encoded_newlines_re + @encoded_newlines_re ||= + Regexp.union(["\r\n", "\r", "\n"].map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) }) + end + + def ruby_encoding + if hash = detect_encoding + hash[:ruby_encoding] + end + end + + def encoding + if hash = detect_encoding + hash[:encoding] + end + end + + def detect_encoding + @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def empty? + data.nil? || data == "" + end + end +end diff --git a/lib/gitlab/branch_push_merge_commit_analyzer.rb b/lib/gitlab/branch_push_merge_commit_analyzer.rb new file mode 100644 index 00000000000..a8f601f2451 --- /dev/null +++ b/lib/gitlab/branch_push_merge_commit_analyzer.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Gitlab + # Analyse a graph of commits from a push to a branch, + # for each commit, analyze that if it is the head of a merge request, + # then what should its merge_commit be, relative to the branch. + # + # A----->B----->C----->D target branch + # | ^ + # | | + # +-->E----->F--+ merged branch + # | ^ + # | | + # +->G--+ + # + # (See merge-commit-analyze-after branch in gitlab-test) + # + # Assuming + # - A is already in remote + # - B~D are all in its own branch with its own merge request, targeting the target branch + # + # When D is finally pushed to the target branch, + # what are the merge commits for all the other merge requests? + # + # We can walk backwards from the HEAD commit D, + # and find status of its parents. + # First we determine if commit belongs to the target branch (i.e. A, B, C, D), + # and then determine its merge commit. + # + # +--------+-----------------+--------------+ + # | Commit | Direct ancestor | Merge commit | + # +--------+-----------------+--------------+ + # | D | Y | D | + # +--------+-----------------+--------------+ + # | C | Y | C | + # +--------+-----------------+--------------+ + # | F | | C | + # +--------+-----------------+--------------+ + # | B | Y | B | + # +--------+-----------------+--------------+ + # | E | | C | + # +--------+-----------------+--------------+ + # | G | | C | + # +--------+-----------------+--------------+ + # + # By examining the result, it can be said that + # + # - If commit is direct ancestor of HEAD, its merge commit is itself. + # - Otherwise, the merge commit is the same as its child's merge commit. + # + class BranchPushMergeCommitAnalyzer + class CommitDecorator < SimpleDelegator + attr_accessor :merge_commit + attr_writer :direct_ancestor # boolean + + def direct_ancestor? + @direct_ancestor + end + + # @param child_commit [CommitDecorator] + # @param first_parent [Boolean] whether `self` is the first parent of `child_commit` + def set_merge_commit(child_commit:) + @merge_commit ||= direct_ancestor? ? self : child_commit.merge_commit + end + end + + # @param commits [Array] list of commits, must be ordered from the child (tip) of the graph back to the ancestors + def initialize(commits, relevant_commit_ids: nil) + @commits = commits + @id_to_commit = {} + @commits.each do |commit| + @id_to_commit[commit.id] = CommitDecorator.new(commit) + + if relevant_commit_ids + relevant_commit_ids.delete(commit.id) + break if relevant_commit_ids.empty? # Only limit the analyze up to relevant_commit_ids + end + end + + analyze + end + + def get_merge_commit(id) + get_commit(id).merge_commit.id + end + + private + + def analyze + head_commit = get_commit(@commits.first.id) + head_commit.direct_ancestor = true + head_commit.merge_commit = head_commit + + mark_all_direct_ancestors(head_commit) + + # Analyzing a commit requires its child commit be analyzed first, + # which is the case here since commits are ordered from child to parent. + @id_to_commit.each_value do |commit| + analyze_parents(commit) + end + end + + def analyze_parents(commit) + commit.parent_ids.each do |parent_commit_id| + parent_commit = get_commit(parent_commit_id) + + next unless parent_commit # parent commit may not be part of new commits + + parent_commit.set_merge_commit(child_commit: commit) + end + end + + # Mark all direct ancestors. + # If child commit is a direct ancestor, its first parent is also a direct ancestor. + # We assume direct ancestors matches the trail of the target branch over time, + # This assumption is correct most of the time, especially for gitlab managed merges, + # but there are exception cases which can't be solved (https://stackoverflow.com/a/49754723/474597) + def mark_all_direct_ancestors(commit) + loop do + commit = get_commit(commit.parent_ids.first) + + break unless commit + + commit.direct_ancestor = true + end + end + + def get_commit(id) + @id_to_commit[id] + end + end +end diff --git a/lib/gitlab/build_access.rb b/lib/gitlab/build_access.rb index 08a8f846ca5..37e79413541 100644 --- a/lib/gitlab/build_access.rb +++ b/lib/gitlab/build_access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class BuildAccess < UserAccess attr_accessor :user, :project diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index add048d671e..ea7013db2ce 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This class is not backed by a table in the main database. # It loads the latest Pipeline for the HEAD of a repository, and caches that # in Redis. @@ -5,9 +7,9 @@ module Gitlab module Cache module Ci class ProjectPipelineStatus - attr_accessor :sha, :status, :ref, :project, :loaded + include Gitlab::Utils::StrongMemoize - delegate :commit, to: :project + attr_accessor :sha, :status, :ref, :project, :loaded def self.load_for_project(project) new(project).tap do |status| @@ -16,33 +18,12 @@ module Gitlab end def self.load_in_batch_for_projects(projects) - cached_results_for_projects(projects).zip(projects).each do |result, project| - project.pipeline_status = new(project, result) + projects.each do |project| + project.pipeline_status = new(project) project.pipeline_status.load_status end end - def self.cached_results_for_projects(projects) - result = Gitlab::Redis::Cache.with do |redis| - redis.multi do - projects.each do |project| - cache_key = cache_key_for_project(project) - redis.exists(cache_key) - redis.hmget(cache_key, :sha, :status, :ref) - end - end - end - - result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| - pipeline_info = { sha: sha, status: status, ref: ref } - { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } - end - end - - def self.cache_key_for_project(project) - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status" - end - def self.update_for_pipeline(pipeline) pipeline_info = { sha: pipeline.sha, @@ -68,6 +49,7 @@ module Gitlab def load_status return if loaded? + return unless commit if has_cache? load_from_cache @@ -80,11 +62,7 @@ module Gitlab end def load_from_project - return unless commit - - self.sha = commit.sha - self.status = commit.status - self.ref = project.default_branch + self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch end # We only cache the status for the HEAD commit of a project @@ -102,6 +80,8 @@ module Gitlab def load_from_cache Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + + self.status = nil if self.status.empty? end end @@ -130,7 +110,13 @@ module Gitlab end def cache_key - self.class.cache_key_for_project(project) + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status:#{commit&.sha}" + end + + def commit + strong_memoize(:commit) do + project.commit + end end end end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..4c658dc0b8d 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,8 @@ +# frozen_string_literal: true + module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self @@ -61,8 +28,8 @@ module Gitlab define_method(method_name) do |*args| store = - if RequestStore.active? - RequestStore.store + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore.store else ivar_name = # ! and ? cannot be used as ivar name "@cache_#{method_name.to_s.tr('!?', "\u2605\u2606")}" diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 9c9e6668e6f..fb75a78a978 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ChangesList include Enumerable diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index e63e5437331..8b3c5dc9e8b 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'json' module Gitlab diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb new file mode 100644 index 00000000000..09b17b5b76b --- /dev/null +++ b/lib/gitlab/checks/base_checker.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BaseChecker + include Gitlab::Utils::StrongMemoize + + attr_reader :change_access + delegate(*ChangeAccess::ATTRIBUTES, to: :change_access) + + def initialize(change_access) + @change_access = change_access + end + + def validate! + raise NotImplementedError + end + + private + + def creation? + Gitlab::Git.blank_ref?(oldrev) + end + + def deletion? + Gitlab::Git.blank_ref?(newrev) + end + + def update? + !creation? && !deletion? + end + + def updated_from_web? + protocol == 'web' + end + + def tag_exists? + project.repository.tag_exists?(tag_name) + end + + def validate_once(resource) + Gitlab::SafeRequestStore.fetch(cache_key_for_resource(resource)) do + yield(resource) + + true + end + end + + def cache_key_for_resource(resource) + "git_access:#{checker_cache_key}:#{resource.cache_key}" + end + + def checker_cache_key + self.class.name.demodulize.underscore + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb new file mode 100644 index 00000000000..d06b2df36f2 --- /dev/null +++ b/lib/gitlab/checks/branch_check.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class BranchCheck < BaseChecker + ERROR_MESSAGES = { + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project maintainer or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.' + }.freeze + + LOG_MESSAGES = { + delete_default_branch_check: "Checking if default branch is being deleted...", + protected_branch_checks: "Checking if you are force pushing to a protected branch...", + protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch..." + }.freeze + + def validate! + return unless branch_name + + logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do + if deletion? && branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end + end + + protected_branch_checks + end + + private + + def protected_branch_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do + return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + if forced_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + end + end + + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do + unless user_access.can_delete_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end + + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end + end + end + + def protected_branch_push_checks + logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do + if matching_merge_request? + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + end + else + unless user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + end + end + end + end + + def push_to_protected_branch_rejected_message + if project.empty_repo? + empty_project_push_message + else + ERROR_MESSAGES[:push_protected_branch] + end + end + + def empty_project_push_message + <<~MESSAGE + + A default branch (e.g. master) does not yet exist for #{project.full_path} + Ask a project Owner or Maintainer to create a default branch: + + #{project_members_url} + + MESSAGE + end + + def project_members_url + Gitlab::Routing.url_helpers.project_project_members_url(project) + end + + def matching_merge_request? + Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? + end + + def forced_push? + Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 51ba09aa129..8a57a3a6d9a 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,192 +1,54 @@ +# frozen_string_literal: true + module Gitlab module Checks class ChangeAccess - ERROR_MESSAGES = { - push_code: 'You are not allowed to push code to this project.', - delete_default_branch: 'The default branch of a project cannot be deleted.', - force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', - non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.', - non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', - merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', - push_protected_branch: 'You are not allowed to push code to protected branches on this project.', - change_existing_tags: 'You are not allowed to change existing tags on this project.', - update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'Protected tags cannot be deleted.', - create_protected_tag: 'You are not allowed to create this tag as it is protected.', - lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' - }.freeze + ATTRIBUTES = %i[user_access project skip_authorization + skip_lfs_integrity_check protocol oldrev newrev ref + branch_name tag_name logger commits].freeze - attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + attr_reader(*ATTRIBUTES) def initialize( - change, user_access:, project:, skip_authorization: false, - skip_lfs_integrity_check: false, protocol: + change, user_access:, project:, + skip_lfs_integrity_check: false, protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project - @skip_authorization = skip_authorization @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol - end - - def exec(skip_commits_check: false) - return true if skip_authorization - - push_checks - branch_checks - tag_checks - lfs_objects_exist_check unless skip_lfs_integrity_check - commits_check unless skip_commits_check - - true - end - - protected - - def push_checks - unless can_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] - end - end - - def branch_checks - return unless branch_name - - if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] - end - - protected_branch_checks - end - - def protected_branch_checks - return unless ProtectedBranch.protected?(project, branch_name) - if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] - end - - if deletion? - protected_branch_deletion_checks - else - protected_branch_push_checks - end + @logger = logger + @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end - def protected_branch_deletion_checks - unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] - end + def exec + ref_level_checks + # Check of commits should happen as the last step + # given they're expensive in terms of performance + commits_check - unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] - end - end - - def protected_branch_push_checks - if matching_merge_request? - unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] - end - else - unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] - end - end + true end - def tag_checks - return unless tag_name - - if tag_exists? && user_access.cannot_do_action?(:admin_project) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] - end - - protected_tag_checks + def commits + @commits ||= project.repository.new_commits(newrev) end - def protected_tag_checks - return unless ProtectedTag.protected?(project, tag_name) - - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + protected - unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] - end + def ref_level_checks + Gitlab::Checks::PushCheck.new(self).validate! + Gitlab::Checks::BranchCheck.new(self).validate! + Gitlab::Checks::TagCheck.new(self).validate! + Gitlab::Checks::LfsCheck.new(self).validate! end def commits_check - return if deletion? || newrev.nil? - return unless should_run_commit_validations? - - # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - commit_check.validate(commit, validations_for_commit(commit)) - end - end - - commit_check.validate_file_paths - end - - # Method overwritten in EE to inject custom validations - def validations_for_commit(_) - [] - end - - private - - def should_run_commit_validations? - commit_check.validate_lfs_file_locks? - end - - def updated_from_web? - protocol == 'web' - end - - def tag_exists? - project.repository.tag_exists?(tag_name) - end - - def forced_push? - Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev) - end - - def update? - !Gitlab::Git.blank_ref?(oldrev) && !deletion? - end - - def deletion? - Gitlab::Git.blank_ref?(newrev) - end - - def matching_merge_request? - Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? - end - - def lfs_objects_exist_check - lfs_check = Checks::LfsIntegrity.new(project, newrev) - - if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] - end - end - - def commit_check - @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) - end - - def commits - @commits ||= project.repository.new_commits(newrev) - end - - def can_push? - user_access.can_do_action?(:push_code) || - user_access.can_push_to_branch?(branch_name) + Gitlab::Checks::DiffCheck.new(self).validate! end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb deleted file mode 100644 index 43a52b493bb..00000000000 --- a/lib/gitlab/checks/commit_check.rb +++ /dev/null @@ -1,61 +0,0 @@ -module Gitlab - module Checks - class CommitCheck - include Gitlab::Utils::StrongMemoize - - attr_reader :project, :user, :newrev, :oldrev - - def initialize(project, user, newrev, oldrev) - @project = project - @user = user - @newrev = user - @oldrev = user - @file_paths = [] - end - - def validate(commit, validations) - return if validations.empty? && path_validations.empty? - - commit.raw_deltas.each do |diff| - @file_paths << (diff.new_path || diff.old_path) - - validations.each do |validation| - if error = validation.call(diff) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - end - - def validate_file_paths - path_validations.each do |validation| - if error = validation.call(@file_paths) - raise ::Gitlab::GitAccess::UnauthorizedError, error - end - end - end - - def validate_lfs_file_locks? - strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev - end - end - - private - - def lfs_file_locks_validation - lambda do |paths| - lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first - - if lfs_lock - return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" - end - end - end - - def path_validations - validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] - end - end - end -end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb new file mode 100644 index 00000000000..ea0d8c85a66 --- /dev/null +++ b/lib/gitlab/checks/diff_check.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class DiffCheck < BaseChecker + include Gitlab::Utils::StrongMemoize + + LOG_MESSAGES = { + validate_file_paths: "Validating diffs' file paths...", + diff_content_check: "Validating diff contents..." + }.freeze + + def validate! + return if deletion? + return unless should_run_diff_validations? + return if commits.empty? + + file_paths = [] + + process_commits do |commit| + validate_once(commit) do + commit.raw_deltas.each do |diff| + file_paths << (diff.new_path || diff.old_path) + + validate_diff(diff) + end + end + end + + validate_file_paths(file_paths) + end + + private + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.any_lfs_file_locks? + end + end + + def should_run_diff_validations? + validations_for_diff.present? || path_validations.present? + end + + def validate_diff(diff) + validations_for_diff.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + # Method overwritten in EE to inject custom validations + def validations_for_diff + [] + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + + def process_commits + logger.log_timed(LOG_MESSAGES[:diff_content_check]) do + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + logger.check_timeout_reached + + yield(commit) + end + end + end + end + + def validate_file_paths(file_paths) + logger.log_timed(LOG_MESSAGES[__method__]) do + path_validations.each do |validation| + if error = validation.call(file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index c9c3050cfc2..263972923ed 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Checks class ForcePush @@ -7,18 +9,10 @@ module Gitlab # Created or deleted branch return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - GitalyClient.migrate(:force_push) do |is_enabled| - if is_enabled - !project - .repository - .gitaly_commit_client - .ancestor?(oldrev, newrev) - else - Gitlab::Git::RevList.new( - project.repository.raw, oldrev: oldrev, newrev: newrev - ).missed_ref.present? - end - end + !project + .repository + .gitaly_commit_client + .ancestor?(oldrev, newrev) end end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb new file mode 100644 index 00000000000..cc6a14d2d9a --- /dev/null +++ b/lib/gitlab/checks/lfs_check.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class LfsCheck < BaseChecker + LOG_MESSAGE = "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...".freeze + ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze + + def validate! + return unless project.lfs_enabled? + return if skip_lfs_integrity_check + + logger.log_timed(LOG_MESSAGE) do + lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) + + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGE + end + end + end + end + end +end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index f0e5773ec3c..1652d5a30a4 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -1,17 +1,20 @@ +# frozen_string_literal: true + module Gitlab module Checks class LfsIntegrity - REV_LIST_OBJECT_LIMIT = 2_000 - - def initialize(project, newrev) + def initialize(project, newrev, time_left) @project = project @newrev = newrev + @time_left = time_left end + # rubocop: disable CodeReuse/ActiveRecord def objects_missing? return false unless @newrev && @project.lfs_enabled? - new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev).new_pointers(object_limit: REV_LIST_OBJECT_LIMIT) + new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev) + .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left) return false unless new_lfs_pointers.present? @@ -21,6 +24,7 @@ module Gitlab existing_count != new_lfs_pointers.count end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 849848515da..71361b12d07 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Checks class MatchingMergeRequest @@ -7,12 +9,14 @@ module Gitlab @project = project end + # rubocop: disable CodeReuse/ActiveRecord def match? @project.merge_requests .with_state(:locked) .where(in_progress_merge_commit_sha: @newrev, target_branch: @branch_name) .exists? end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb index 473c0385b34..492dbb5a596 100644 --- a/lib/gitlab/checks/post_push_message.rb +++ b/lib/gitlab/checks/post_push_message.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Checks class PostPushMessage diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index cec270d6a58..0058a402a62 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Checks class ProjectCreated < PostPushMessage diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3a197078d08..cb3b7acaaad 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Checks class ProjectMoved < PostPushMessage diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb new file mode 100644 index 00000000000..91f8d0bdbc8 --- /dev/null +++ b/lib/gitlab/checks/push_check.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class PushCheck < BaseChecker + def validate! + logger.log_timed("Checking if you are allowed to push...") do + unless can_push? + raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code] + end + end + end + + private + + def can_push? + user_access.can_do_action?(:push_code) || + project.branch_allows_collaboration?(user_access.user, branch_name) + end + end + end +end diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb new file mode 100644 index 00000000000..2a75c8059bd --- /dev/null +++ b/lib/gitlab/checks/tag_check.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TagCheck < BaseChecker + ERROR_MESSAGES = { + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + + LOG_MESSAGES = { + tag_checks: "Checking if you are allowed to change existing tags...", + protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag..." + }.freeze + + def validate! + return unless tag_name + + logger.log_timed(LOG_MESSAGES[:tag_checks]) do + if tag_exists? && user_access.cannot_do_action?(:admin_project) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + end + end + + protected_tag_checks + end + + private + + def protected_tag_checks + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks + + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + + unless user_access.can_create_tag?(tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + end + end + end + end + end +end diff --git a/lib/gitlab/checks/timed_logger.rb b/lib/gitlab/checks/timed_logger.rb new file mode 100644 index 00000000000..f365e0a43f6 --- /dev/null +++ b/lib/gitlab/checks/timed_logger.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TimedLogger + TimeoutError = Class.new(StandardError) + + attr_reader :start_time, :header, :log, :timeout + + def initialize(start_time: Time.now, log: [], header: "", timeout:) + @start_time = start_time + @timeout = timeout + @header = header + @log = log + end + + # Adds trace of method being tracked with + # the correspondent time it took to run it. + # We make use of the start default argument + # on unit tests related to this method + # + def log_timed(log_message, start = Time.now) + check_timeout_reached + + timed = true + + yield + + append_message(log_message + time_suffix_message(start: start)) + rescue GRPC::DeadlineExceeded, TimeoutError + args = { cancelled: true } + args[:start] = start if timed + + append_message(log_message + time_suffix_message(args)) + + raise TimeoutError + end + + def check_timeout_reached + return unless time_expired? + + raise TimeoutError + end + + def time_left + (start_time + timeout.seconds) - Time.now + end + + def full_message + header + log.join("\n") + end + + # We always want to append in-place on the log + def append_message(message) + log << message + end + + private + + def time_expired? + time_left <= 0 + end + + def time_suffix_message(cancelled: false, start: nil) + return " (#{elapsed_time(start)}ms)" unless cancelled + + if start + " (cancelled after #{elapsed_time(start)}ms)" + else + " (cancelled)" + end + end + + def elapsed_time(start) + to_ms(Time.now - start) + end + + def to_ms(elapsed) + (elapsed.to_f * 1000).round(2) + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 35eadf6fa93..4dcb3869d4f 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # ANSI color library # # Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code @@ -29,105 +31,105 @@ module Gitlab end class Converter - def on_0(s) reset() end + def on_0(_) reset end - def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_1(_) enable(STYLE_SWITCHES[:bold]) end - def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_3(_) enable(STYLE_SWITCHES[:italic]) end - def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_4(_) enable(STYLE_SWITCHES[:underline]) end - def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_8(_) enable(STYLE_SWITCHES[:conceal]) end - def on_9(s) enable(STYLE_SWITCHES[:cross]) end + def on_9(_) enable(STYLE_SWITCHES[:cross]) end - def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_21(_) disable(STYLE_SWITCHES[:bold]) end - def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(_) disable(STYLE_SWITCHES[:bold]) end - def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_23(_) disable(STYLE_SWITCHES[:italic]) end - def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_24(_) disable(STYLE_SWITCHES[:underline]) end - def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_28(_) disable(STYLE_SWITCHES[:conceal]) end - def on_29(s) disable(STYLE_SWITCHES[:cross]) end + def on_29(_) disable(STYLE_SWITCHES[:cross]) end - def on_30(s) set_fg_color(0) end + def on_30(_) set_fg_color(0) end - def on_31(s) set_fg_color(1) end + def on_31(_) set_fg_color(1) end - def on_32(s) set_fg_color(2) end + def on_32(_) set_fg_color(2) end - def on_33(s) set_fg_color(3) end + def on_33(_) set_fg_color(3) end - def on_34(s) set_fg_color(4) end + def on_34(_) set_fg_color(4) end - def on_35(s) set_fg_color(5) end + def on_35(_) set_fg_color(5) end - def on_36(s) set_fg_color(6) end + def on_36(_) set_fg_color(6) end - def on_37(s) set_fg_color(7) end + def on_37(_) set_fg_color(7) end - def on_38(s) set_fg_color_256(s) end + def on_38(stack) set_fg_color_256(stack) end - def on_39(s) set_fg_color(9) end + def on_39(_) set_fg_color(9) end - def on_40(s) set_bg_color(0) end + def on_40(_) set_bg_color(0) end - def on_41(s) set_bg_color(1) end + def on_41(_) set_bg_color(1) end - def on_42(s) set_bg_color(2) end + def on_42(_) set_bg_color(2) end - def on_43(s) set_bg_color(3) end + def on_43(_) set_bg_color(3) end - def on_44(s) set_bg_color(4) end + def on_44(_) set_bg_color(4) end - def on_45(s) set_bg_color(5) end + def on_45(_) set_bg_color(5) end - def on_46(s) set_bg_color(6) end + def on_46(_) set_bg_color(6) end - def on_47(s) set_bg_color(7) end + def on_47(_) set_bg_color(7) end - def on_48(s) set_bg_color_256(s) end + def on_48(stack) set_bg_color_256(stack) end - def on_49(s) set_bg_color(9) end + def on_49(_) set_bg_color(9) end - def on_90(s) set_fg_color(0, 'l') end + def on_90(_) set_fg_color(0, 'l') end - def on_91(s) set_fg_color(1, 'l') end + def on_91(_) set_fg_color(1, 'l') end - def on_92(s) set_fg_color(2, 'l') end + def on_92(_) set_fg_color(2, 'l') end - def on_93(s) set_fg_color(3, 'l') end + def on_93(_) set_fg_color(3, 'l') end - def on_94(s) set_fg_color(4, 'l') end + def on_94(_) set_fg_color(4, 'l') end - def on_95(s) set_fg_color(5, 'l') end + def on_95(_) set_fg_color(5, 'l') end - def on_96(s) set_fg_color(6, 'l') end + def on_96(_) set_fg_color(6, 'l') end - def on_97(s) set_fg_color(7, 'l') end + def on_97(_) set_fg_color(7, 'l') end - def on_99(s) set_fg_color(9, 'l') end + def on_99(_) set_fg_color(9, 'l') end - def on_100(s) set_bg_color(0, 'l') end + def on_100(_) set_bg_color(0, 'l') end - def on_101(s) set_bg_color(1, 'l') end + def on_101(_) set_bg_color(1, 'l') end - def on_102(s) set_bg_color(2, 'l') end + def on_102(_) set_bg_color(2, 'l') end - def on_103(s) set_bg_color(3, 'l') end + def on_103(_) set_bg_color(3, 'l') end - def on_104(s) set_bg_color(4, 'l') end + def on_104(_) set_bg_color(4, 'l') end - def on_105(s) set_bg_color(5, 'l') end + def on_105(_) set_bg_color(5, 'l') end - def on_106(s) set_bg_color(6, 'l') end + def on_106(_) set_bg_color(6, 'l') end - def on_107(s) set_bg_color(7, 'l') end + def on_107(_) set_bg_color(7, 'l') end - def on_109(s) set_bg_color(9, 'l') end + def on_109(_) set_bg_color(9, 'l') end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask @@ -175,7 +177,7 @@ module Gitlab end end - close_open_tags() + close_open_tags OpenStruct.new( html: @out.force_encoding(Encoding.default_external), @@ -188,29 +190,29 @@ module Gitlab ) end - def handle_section(s) - action = s[1] - timestamp = s[2] - section = s[3] - line = s.matched()[0...-5] # strips \r\033[0K + def handle_section(scanner) + action = scanner[1] + timestamp = scanner[2] + section = scanner[3] + line = scanner.matched[0...-5] # strips \r\033[0K @out << %{<div class="hidden" data-action="#{action}" data-timestamp="#{timestamp}" data-section="#{section}">#{line}</div>} end - def handle_sequence(s) - indicator = s[1] - commands = s[2].split ';' - terminator = s[3] + def handle_sequence(scanner) + indicator = scanner[1] + commands = scanner[2].split ';' + terminator = scanner[3] # We are only interested in color and text style changes - triggered by # sequences starting with '\e[' and ending with 'm'. Any other control # sequence gets stripped (including stuff like "delete last line") return unless indicator == '[' && terminator == 'm' - close_open_tags() + close_open_tags - if commands.empty?() - reset() + if commands.empty? + reset return end @@ -220,7 +222,7 @@ module Gitlab end def evaluate_command_stack(stack) - return unless command = stack.shift() + return unless command = stack.shift if self.respond_to?("on_#{command}", true) self.__send__("on_#{command}", stack) # rubocop:disable GitlabSecurity/PublicSend @@ -265,7 +267,7 @@ module Gitlab def reset_state @offset = 0 @n_open_tags = 0 - @out = '' + @out = +'' reset end @@ -331,8 +333,8 @@ module Gitlab return unless command_stack.length >= 2 return unless command_stack[0] == "5" - command_stack.shift() # ignore the "5" command - color_index = command_stack.shift().to_i + command_stack.shift # ignore the "5" command + color_index = command_stack.shift.to_i return unless color_index >= 0 return unless color_index <= 255 diff --git a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb new file mode 100644 index 00000000000..25a82086676 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Artifacts + module Adapters + class GzipStream + attr_reader :stream + + InvalidStreamError = Class.new(StandardError) + + def initialize(stream) + raise InvalidStreamError, "Stream is required" unless stream + + @stream = stream + end + + def each_blob + stream.seek(0) + + until stream.eof? + gzip(stream) do |gz| + yield gz.read, gz.orig_name + unused = gz.unused&.length.to_i + # pos has already reached to EOF at the moment + # We rewind the pos to the top of unused files + # to read next gzip stream, to support multistream archives + # https://golang.org/src/compress/gzip/gunzip.go#L117 + stream.seek(-unused, IO::SEEK_CUR) + end + end + end + + private + + def gzip(stream, &block) + gz = Zlib::GzipReader.new(stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb new file mode 100644 index 00000000000..cf37d700991 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Artifacts + module Adapters + class RawStream + attr_reader :stream + + InvalidStreamError = Class.new(StandardError) + + def initialize(stream) + raise InvalidStreamError, "Stream is required" unless stream + + @stream = stream + end + + def each_blob + stream.seek(0) + + yield(stream.read, 'raw') unless stream.eof? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 0bbd60d8ffe..08dac756cc1 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'zlib' require 'json' @@ -7,14 +9,15 @@ module Gitlab module Artifacts class Metadata ParserError = Class.new(StandardError) + InvalidStreamError = Class.new(StandardError) VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} - attr_reader :file, :path, :full_version + attr_reader :stream, :path, :full_version - def initialize(file, path, **opts) - @file, @path, @opts = file, path, opts + def initialize(stream, path, **opts) + @stream, @path, @opts = stream, path, opts @full_version = read_version end @@ -58,9 +61,12 @@ module Gitlab until gz.eof? begin - path = read_string(gz).force_encoding('UTF-8') - meta = read_string(gz).force_encoding('UTF-8') + path = read_string(gz)&.force_encoding('UTF-8') + meta = read_string(gz)&.force_encoding('UTF-8') + # We might hit an EOF while reading either value, so we should + # abort if we don't get any data. + next unless path && meta next unless path.valid_encoding? && meta.valid_encoding? next unless path =~ match_pattern next if path =~ INVALID_PATH_PATTERN @@ -103,7 +109,17 @@ module Gitlab end def gzip(&block) - Zlib::GzipReader.open(@file, &block) + raise InvalidStreamError, "Invalid stream" unless @stream + + # restart gzip reading + @stream.seek(0) + + gz = Zlib::GzipReader.new(@stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 428c0505808..d0a80518ae8 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build @@ -96,12 +98,14 @@ module Gitlab blank_node? || @entries.include?(@path.to_s) end + # rubocop: disable CodeReuse/ActiveRecord def total_size descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end end + # rubocop: enable CodeReuse/ActiveRecord def path @path.to_s diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb index 9cd9b36c5f8..65cd935afaa 100644 --- a/lib/gitlab/ci/build/artifacts/path.rb +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 29a7a27c963..58adf6e506d 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index 2423aa8857d..fa805abb8bb 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb index 55eafcaed10..1c8588d9913 100644 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index c811f88f483..4dd932f61d4 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb index d10cc7802d4..43c46ad74af 100644 --- a/lib/gitlab/ci/build/policy.rb +++ b/lib/gitlab/ci/build/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb new file mode 100644 index 00000000000..1663c875426 --- /dev/null +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Policy + class Changes < Policy::Specification + def initialize(globs) + @globs = Array(globs) + end + + def satisfied_by?(pipeline, seed) + return true unless pipeline.branch_updated? + + pipeline.modified_paths.any? do |path| + @globs.any? do |glob| + File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index 782f6c4c0af..4c7dc947cd0 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 4aa5dc89f47..0e9bb5c94bb 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build @@ -30,10 +32,14 @@ module Gitlab return true if pipeline.source == pattern return true if pipeline.source&.pluralize == pattern - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ pipeline.ref - else - pattern == pipeline.ref + # patterns can be matched only when branch or tag is used + # the pattern matching does not work for merge requests pipelines + if pipeline.branch? || pipeline.tag? + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ pipeline.ref + else + pattern == pipeline.ref + end end end end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index f09ba42c074..ceb5210cfb5 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 9d2a362b7d4..0698136166a 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 0b1ebe4e048..7fcabc035ac 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build @@ -13,7 +15,6 @@ module Gitlab def from_commands(job) self.new(:script).tap do |step| step.script = job.options[:before_script].to_a + job.options[:script].to_a - step.script = job.commands.split("\n") if step.script.empty? step.timeout = job.metadata_timeout step.when = WHEN_ON_SUCCESS end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 46ed330dbbf..7cabaadb122 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -1,13 +1,17 @@ +# frozen_string_literal: true + module Gitlab module Ci module Charts module DailyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) query .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.day @@ -15,6 +19,7 @@ module Gitlab end module MonthlyInterval + # rubocop: disable CodeReuse/ActiveRecord def grouped_count(query) if Gitlab::Database.postgresql? query @@ -27,6 +32,7 @@ module Gitlab .count(:created_at) end end + # rubocop: enable CodeReuse/ActiveRecord def interval_step @interval_step ||= 1.month @@ -46,8 +52,9 @@ module Gitlab collect end + # rubocop: disable CodeReuse/ActiveRecord def collect - query = project.pipelines + query = project.all_pipelines .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection totals_count = grouped_count(query) @@ -64,6 +71,7 @@ module Gitlab current += interval_step end end + # rubocop: enable CodeReuse/ActiveRecord end class YearChart < Chart @@ -107,7 +115,7 @@ module Gitlab class PipelineTime < Chart def collect - commits = project.pipelines.last(30) + commits = project.all_pipelines.last(30) commits.each do |commit| @labels << commit.short_sha diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 66ac4a40616..5875479183e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,15 +1,24 @@ +# frozen_string_literal: true + module Gitlab module Ci - ## + # # Base GitLab CI Configuration facade # class Config - # EE would override this and utilize opts argument - def initialize(config, opts = {}) - @config = Loader.new(config).load! + ConfigError = Class.new(StandardError) + + def initialize(config, project: nil, sha: nil, user: nil) + @config = Config::Extendable + .new(build_config(config, project: project, sha: sha, user: user)) + .to_hash @global = Entry::Global.new(@config) @global.compose! + rescue Gitlab::Config::Loader::FormatError, + Extendable::ExtensionError, + External::Processor::IncludeError => e + raise Config::ConfigError, e.message end def valid? @@ -58,6 +67,25 @@ module Gitlab def jobs @global.jobs_value end + + private + + def build_config(config, project:, sha:, user:) + initial_config = Gitlab::Config::Loader::Yaml.new(config).load! + + if project + process_external_files(initial_config, project: project, sha: sha, user: user) + else + initial_config + end + end + + def process_external_files(config, project:, sha:, user:) + Config::External::Processor.new(config, + project: project, + sha: sha || project.repository.root_ref_sha, + user: user).perform + end end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 8275aacee9b..41613369ca2 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,14 +7,17 @@ module Gitlab ## # Entry that represents a configuration of job artifacts. # - class Artifacts < Node - include Validatable - include Attributable + class Artifacts < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in].freeze attributes ALLOWED_KEYS + entry :reports, Entry::Reports, description: 'Report-type artifacts.' + validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS @@ -21,6 +26,7 @@ module Gitlab validates :name, type: String validates :untracked, boolean: true validates :paths, array_of_strings: true + validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ @@ -28,6 +34,13 @@ module Gitlab validates :expire_in, duration: true end end + + helpers :reports + + def value + @config[:reports] = reports_value if @config.key?(:reports) + @config + end end end end diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb deleted file mode 100644 index 3e87a09704e..00000000000 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - module Attributable - extend ActiveSupport::Concern - - class_methods do - def attributes(*attributes) - attributes.flatten.each do |attribute| - if method_defined?(attribute) - raise ArgumentError, 'Method already defined!' - end - - define_method(attribute) do - return unless config.is_a?(Hash) - - config[attribute] - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb deleted file mode 100644 index f3357f85b99..00000000000 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents a boolean value. - # - class Boolean < Node - include Validatable - - validations do - validates :config, boolean: true - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index d7e09acbbf3..7b94af24c09 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,9 +7,9 @@ module Gitlab ## # Entry that represents a cache configuration # - class Cache < Node - include Configurable - include Attributable + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[key untracked paths policy].freeze DEFAULT_POLICY = 'pull-push'.freeze @@ -20,7 +22,7 @@ module Gitlab entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' - entry :untracked, Entry::Boolean, + entry :untracked, ::Gitlab::Config::Entry::Boolean, description: 'Cache all untracked files.' entry :paths, Entry::Paths, diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 65d19db249c..02e368c1813 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,22 +7,11 @@ module Gitlab ## # Entry that represents a job script. # - class Commands < Node - include Validatable + class Commands < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do - include LegacyValidationHelpers - - validate do - unless string_or_array_of_strings?(config) - errors.add(:config, - 'should be a string or an array of strings') - end - end - - def string_or_array_of_strings?(field) - validate_string(field) || validate_array_of_strings(field) - end + validates :config, array_of_strings_or_string: true end def value diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb deleted file mode 100644 index db47c2f6185..00000000000 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ /dev/null @@ -1,77 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # This mixin is responsible for adding DSL, which purpose is to - # simplifly process of adding child nodes. - # - # This can be used only if parent node is a configuration entry that - # holds a hash as a configuration value, for example: - # - # job: - # script: ... - # artifacts: ... - # - module Configurable - extend ActiveSupport::Concern - - included do - include Validatable - - validations do - validates :config, type: Hash - end - end - - def compose!(deps = nil) - return unless valid? - - self.class.nodes.each do |key, factory| - factory - .value(config[key]) - .with(key: key, parent: self) - - entries[key] = factory.create! - end - - yield if block_given? - - entries.each_value do |entry| - entry.compose!(deps) - end - end - - class_methods do - def nodes - Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] - end - - private # rubocop:disable Lint/UselessAccessModifier - - def entry(key, entry, metadata) - factory = Entry::Factory.new(entry) - .with(description: metadata[:description]) - - (@nodes ||= {}).merge!(key.to_sym => factory) - end - - def helpers(*nodes) - nodes.each do |symbol| - define_method("#{symbol}_defined?") do - entries[symbol]&.specified? - end - - define_method("#{symbol}_value") do - return unless entries[symbol] && entries[symbol].valid? - - entries[symbol].value - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb index 12a063059cb..89545158bed 100644 --- a/lib/gitlab/ci/config/entry/coverage.rb +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents Coverage settings. # - class Coverage < Node - include Validatable + class Coverage < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, regexp: true diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 0c1f9eb7cbf..69a3a1aedef 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents an environment. # - class Environment < Node - include Validatable + class Environment < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name url action on_stop].freeze diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb deleted file mode 100644 index 6be8288748f..00000000000 --- a/lib/gitlab/ci/config/entry/factory.rb +++ /dev/null @@ -1,73 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Factory class responsible for fabricating entry objects. - # - class Factory - InvalidFactory = Class.new(StandardError) - - def initialize(entry) - @entry = entry - @metadata = {} - @attributes = {} - end - - def value(value) - @value = value - self - end - - def metadata(metadata) - @metadata.merge!(metadata) - self - end - - def with(attributes) - @attributes.merge!(attributes) - self - end - - def create! - raise InvalidFactory unless defined?(@value) - - ## - # We assume that unspecified entry is undefined. - # See issue #18775. - # - if @value.nil? - Entry::Unspecified.new( - fabricate_unspecified - ) - else - fabricate(@entry, @value) - end - end - - private - - def fabricate_unspecified - ## - # If entry has a default value we fabricate concrete node - # with default value. - # - if @entry.default.nil? - fabricate(Entry::Undefined) - else - fabricate(@entry, @entry.default) - end - end - - def fabricate(entry, value = nil) - entry.new(value, @metadata).tap do |node| - node.key = @attributes[:key] - node.parent = @attributes[:parent] - node.description = @attributes[:description] - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index a4ec8f0ff2f..09ecb5fdb99 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -6,8 +8,8 @@ module Gitlab # This class represents a global entry - root Entry for entire # GitLab CI Configuration file. # - class Global < Node - include Configurable + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable entry :before_script, Entry::Script, description: 'Script that will be executed before each job.' @@ -45,14 +47,16 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def compose_jobs! - factory = Entry::Factory.new(Entry::Jobs) + factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) .with(key: :jobs, parent: self, description: 'Jobs definition for this pipeline') @entries[:jobs] = factory.create! end + # rubocop: enable CodeReuse/ActiveRecord def compose_deprecated_entries! ## diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index 6fc3aa385bc..76e5d05639f 100644 --- a/lib/gitlab/ci/config/entry/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a hidden CI/CD key. # - class Hidden < Node - include Validatable + class Hidden < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, presence: true diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 2844be80a84..a13a0625e90 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a Docker image. # - class Image < Node - include Validatable + class Image < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name entrypoint].freeze diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 91aac6df4b1..290c9591b98 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,13 +7,14 @@ module Gitlab ## # Entry that represents a concrete CI/CD job. # - class Job < Node - include Configurable - include Attributable + class Job < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[tags script only except type image services allow_failure - type stage when artifacts cache dependencies before_script - after_script variables environment coverage retry].freeze + ALLOWED_KEYS = %i[tags script only except type image services + allow_failure type stage when start_in artifacts cache + dependencies before_script after_script variables + environment coverage retry parallel extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -23,16 +26,19 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :retry, numericality: { only_integer: true, - greater_than_or_equal_to: 0, - less_than_or_equal_to: 2 } + validates :parallel, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: 50 } validates :when, - inclusion: { in: %w[on_success on_failure always manual], + inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ - 'always or manual' } - + 'always, manual or delayed' } validates :dependencies, array_of_strings: true + validates :extends, type: String end + + validates :start_in, duration: { limit: '1 day' }, if: :delayed? + validates :start_in, absence: true, unless: :delayed? end entry :before_script, Entry::Script, @@ -60,7 +66,8 @@ module Gitlab description: 'Services that will be used to execute this job.' entry :only, Entry::Policy, - description: 'Refs policy this job will be executed for.' + description: 'Refs policy this job will be executed for.', + default: Entry::Policy::DEFAULT_ONLY entry :except, Entry::Policy, description: 'Refs policy this job will be executed for.' @@ -72,16 +79,21 @@ module Gitlab description: 'Artifacts configuration for this job.' entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.' entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.' + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry + :artifacts, :environment, :coverage, :retry, + :parallel - attributes :script, :tags, :allow_failure, :when, :dependencies, :retry + attributes :script, :tags, :allow_failure, :when, :dependencies, + :retry, :parallel, :extends, :start_in def compose!(deps = nil) super do @@ -103,14 +115,14 @@ module Gitlab @config.merge(to_hash.compact) end - def commands - (before_script_value.to_a + script_value.to_a).join("\n") - end - def manual_action? self.when == 'manual' end + def delayed? + self.when == 'delayed' + end + def ignored? allow_failure.nil? ? manual_action? : allow_failure end @@ -134,7 +146,6 @@ module Gitlab { name: name, before_script: before_script_value, script: script_value, - commands: commands, image: image_value, services: services_value, stage: stage_value, @@ -145,7 +156,8 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, - retry: retry_defined? ? retry_value.to_i : nil, + retry: retry_defined? ? retry_value : nil, + parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 5671a09480b..9845c4af655 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a set of jobs. # - class Jobs < Node - include Validatable + class Jobs < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: Hash @@ -26,12 +28,17 @@ module Gitlab name.to_s.start_with?('.') end + def node_type(name) + hidden?(name) ? Entry::Hidden : Entry::Job + end + + # rubocop: disable CodeReuse/ActiveRecord def compose!(deps = nil) super do @config.each do |name, config| - node = hidden?(name) ? Entry::Hidden : Entry::Job + node = node_type(name) - factory = Entry::Factory.new(node) + factory = ::Gitlab::Config::Entry::Factory.new(node) .value(config || {}) .metadata(name: name) .with(key: name, parent: self, @@ -45,6 +52,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index f27ad0a7759..0c10967e629 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a key. # - class Key < Node - include Validatable + class Key < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, key: true diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb deleted file mode 100644 index a78a85397bd..00000000000 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ /dev/null @@ -1,61 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - module LegacyValidationHelpers - private - - def validate_duration(value) - value.is_a?(String) && ChronicDuration.parse(value) - rescue ChronicDuration::DurationParseError - false - end - - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_array_of_strings_or_regexps(values) - values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && - variables.flatten.all? do |value| - validate_string(value) || validate_integer(value) - end - end - - def validate_integer(value) - value.is_a?(Integer) - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_regexp(value) - !value.nil? && Regexp.new(value.to_s) && true - rescue RegexpError, TypeError - false - end - - def validate_string_or_regexp(value) - return true if value.is_a?(Symbol) - return false unless value.is_a?(String) - - if value.first == '/' && value.last == '/' - validate_regexp(value[1...-1]) - else - true - end - end - - def validate_boolean(value) - value.in?([true, false]) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb deleted file mode 100644 index 26505c91be3..00000000000 --- a/lib/gitlab/ci/config/entry/node.rb +++ /dev/null @@ -1,101 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # Base abstract class for each configuration entry node. - # - class Node - InvalidError = Class.new(StandardError) - - attr_reader :config, :metadata - attr_accessor :key, :parent, :description - - def initialize(config, **metadata) - @config = config - @metadata = metadata - @entries = {} - - self.class.aspects.to_a.each do |aspect| - instance_exec(&aspect) - end - end - - def [](key) - @entries[key] || Entry::Undefined.new - end - - def compose!(deps = nil) - return unless valid? - - yield if block_given? - end - - def leaf? - @entries.none? - end - - def descendants - @entries.values - end - - def ancestors - @parent ? @parent.ancestors + [@parent] : [] - end - - def valid? - errors.none? - end - - def errors - [] - end - - def value - if leaf? - @config - else - meaningful = @entries.select do |_key, value| - value.specified? && value.relevant? - end - - Hash[meaningful.map { |key, entry| [key, entry.value] }] - end - end - - def specified? - true - end - - def relevant? - true - end - - def location - name = @key.presence || self.class.name.to_s.demodulize - .underscore.humanize.downcase - - ancestors.map(&:key).append(name).compact.join(':') - end - - def inspect - val = leaf? ? config : descendants - unspecified = specified? ? '' : '(unspecified) ' - "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" - end - - def self.default - end - - def self.aspects - @aspects ||= [] - end - - private - - attr_reader :entries - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 68dad161149..d6f287c6552 100644 --- a/lib/gitlab/ci/config/entry/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents an array of paths. # - class Paths < Node - include Validatable + class Paths < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 09e8e52b60f..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,12 +7,14 @@ module Gitlab ## # Entry that represents an only/except trigger policy for the job. # - class Policy < Simplifiable + class Policy < ::Gitlab::Config::Entry::Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } - class RefsPolicy < Entry::Node - include Entry::Validatable + DEFAULT_ONLY = { refs: %w[branches tags] }.freeze + + class RefsPolicy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings_or_regexps: true @@ -21,21 +25,23 @@ module Gitlab end end - class ComplexPolicy < Entry::Node - include Entry::Validatable - include Entry::Attributable + class ComplexPolicy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable - attributes :refs, :kubernetes, :variables + ALLOWED_KEYS = %i[refs kubernetes variables changes].freeze + attributes :refs, :kubernetes, :variables, :changes validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes variables] + validates :config, allowed_keys: ALLOWED_KEYS validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true + validates :changes, array_of_strings: true end def variables_expressions_syntax @@ -54,13 +60,14 @@ module Gitlab end end - class UnknownStrategy < Entry::Node + class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} has to be either an array of conditions or a hash"] end end - def self.default + def value + default.to_h.deep_merge(subject.value.to_h) end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb new file mode 100644 index 00000000000..a3f6cc31321 --- /dev/null +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of job artifacts. + # + class Reports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :junit, array_of_strings_or_string: true + validates :codequality, array_of_strings_or_string: true + validates :sast, array_of_strings_or_string: true + validates :dependency_scanning, array_of_strings_or_string: true + validates :container_scanning, array_of_strings_or_string: true + validates :dast, array_of_strings_or_string: true + validates :performance, array_of_strings_or_string: true + validates :license_management, array_of_strings_or_string: true + end + end + + def value + @config.transform_values { |v| Array(v) } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb new file mode 100644 index 00000000000..e9cbcb31e21 --- /dev/null +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a retry config for a job. + # + class Retry < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) } + strategy :FullRetry, if: -> (config) { config.is_a?(Hash) } + + class SimpleRetry < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + end + + def value + { + max: config + } + end + + def location + 'retry' + end + end + + class FullRetry < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[max when].freeze + attributes :max, :when + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :max, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + + validates :when, array_of_strings_or_string: true + validates :when, + allowed_array_values: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(Array) } + validates :when, + inclusion: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(String) } + end + end + + def self.possible_retry_when_values + @possible_retry_when_values ||= ::Ci::Build.failure_reasons.keys.map(&:to_s) + ['always'] + end + + def value + super.tap do |config| + # make sure that `when` is an array, because we allow it to + # be passed as a String in config for simplicity + config[:when] = Array.wrap(config[:when]) if config[:when] + end + end + + def location + 'retry' + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} has to be either an integer or a hash"] + end + + def location + 'retry config' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index 29ecd9995ca..9d25a82b521 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a script. # - class Script < Node - include Validatable + class Script < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 3e2ebcff31a..6df67083310 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -6,7 +8,7 @@ module Gitlab # Entry that represents a configuration of Docker service. # class Service < Image - include Validatable + include ::Gitlab::Config::Entry::Validatable ALLOWED_KEYS = %i[name entrypoint command alias].freeze diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 0066894e069..71475f69218 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a configuration of Docker services. # - class Services < Node - include Validatable + class Services < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: Array @@ -16,7 +18,7 @@ module Gitlab super do @entries = [] @config.each do |config| - @entries << Entry::Factory.new(Entry::Service) + @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service) .value(config || {}) .create! end diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb deleted file mode 100644 index 12764629686..00000000000 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ /dev/null @@ -1,43 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - class Simplifiable < SimpleDelegator - EntryStrategy = Struct.new(:name, :condition) - - def initialize(config, **metadata) - unless self.class.const_defined?(:UnknownStrategy) - raise ArgumentError, 'UndefinedStrategy not available!' - end - - strategy = self.class.strategies.find do |variant| - variant.condition.call(config) - end - - entry = self.class.entry_class(strategy) - - super(entry.new(config, metadata)) - end - - def self.strategy(name, **opts) - EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| - strategies.append(strategy) - end - end - - def self.strategies - @strategies ||= [] - end - - def self.entry_class(strategy) - if strategy.present? - self.const_get(strategy.name) - else - self::UnknownStrategy - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index b7afaba1de8..d6d576a3139 100644 --- a/lib/gitlab/ci/config/entry/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a stage for a job. # - class Stage < Node - include Validatable + class Stage < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, type: String diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index ec187bd3732..2d715cbc6bb 100644 --- a/lib/gitlab/ci/config/entry/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,8 +7,8 @@ module Gitlab ## # Entry that represents a configuration for pipeline stages. # - class Stages < Node - include Validatable + class Stages < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb deleted file mode 100644 index 1171ac10f22..00000000000 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ /dev/null @@ -1,40 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # This class represents an undefined entry. - # - class Undefined < Node - def initialize(*) - super(nil) - end - - def value - nil - end - - def valid? - true - end - - def errors - [] - end - - def specified? - false - end - - def relevant? - false - end - - def inspect - "#<#{self.class.name}>" - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb deleted file mode 100644 index fbb2551e870..00000000000 --- a/lib/gitlab/ci/config/entry/unspecified.rb +++ /dev/null @@ -1,19 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - ## - # This class represents an unspecified entry. - # - # It decorates original entry adding method that indicates it is - # unspecified. - # - class Unspecified < SimpleDelegator - def specified? - false - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb deleted file mode 100644 index e45787773a8..00000000000 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ /dev/null @@ -1,38 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - module Validatable - extend ActiveSupport::Concern - - def self.included(node) - node.aspects.append -> do - @validator = self.class.validator.new(self) - @validator.validate(:new) - end - end - - def errors - @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - - class_methods do - def validator - @validator ||= Class.new(Entry::Validator).tap do |validator| - if defined?(@validations) - @validations.each { |rules| validator.class_eval(&rules) } - end - end - end - - private - - def validations(&block) - (@validations ||= []).append(block) - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb deleted file mode 100644 index 2df23a3edcd..00000000000 --- a/lib/gitlab/ci/config/entry/validator.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - class Validator < SimpleDelegator - include ActiveModel::Validations - include Entry::Validators - - def initialize(entry) - super(entry) - end - - def messages - errors.full_messages.map do |error| - "#{location} #{error}".downcase - end - end - - def self.name - 'Validator' - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb deleted file mode 100644 index 55658900628..00000000000 --- a/lib/gitlab/ci/config/entry/validators.rb +++ /dev/null @@ -1,158 +0,0 @@ -module Gitlab - module Ci - class Config - module Entry - module Validators - class AllowedKeysValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unknown_keys = record.config.try(:keys).to_a - options[:in] - - if unknown_keys.any? - record.errors.add(:config, 'contains unknown keys: ' + - unknown_keys.join(', ')) - end - end - end - - class AllowedValuesValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless options[:in].include?(value.to_s) - record.errors.add(attribute, "unknown value: #{value}") - end - end - end - - class ArrayOfStringsValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_array_of_strings(value) - record.errors.add(attribute, 'should be an array of strings') - end - end - end - - class BooleanValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_boolean(value) - record.errors.add(attribute, 'should be a boolean value') - end - end - end - - class DurationValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_duration(value) - record.errors.add(attribute, 'should be a duration') - end - end - end - - class HashOrStringValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless value.is_a?(Hash) || value.is_a?(String) - record.errors.add(attribute, 'should be a hash or a string') - end - end - end - - class KeyValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - if validate_string(value) - validate_path(record, attribute, value) - else - record.errors.add(attribute, 'should be a string or symbol') - end - end - - private - - def validate_path(record, attribute, value) - path = CGI.unescape(value.to_s) - - if path.include?('/') - record.errors.add(attribute, 'cannot contain the "/" character') - elsif path == '.' || path == '..' - record.errors.add(attribute, 'cannot be "." or ".."') - end - end - end - - class RegexpValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_regexp(value) - record.errors.add(attribute, 'must be a regular expression') - end - end - - private - - def look_like_regexp?(value) - value.is_a?(String) && value.start_with?('/') && - value.end_with?('/') - end - - def validate_regexp(value) - look_like_regexp?(value) && - Regexp.new(value.to_s[1...-1]) && - true - rescue RegexpError - false - end - end - - class ArrayOfStringsOrRegexpsValidator < RegexpValidator - def validate_each(record, attribute, value) - unless validate_array_of_strings_or_regexps(value) - record.errors.add(attribute, 'should be an array of strings or regexps') - end - end - - private - - def validate_array_of_strings_or_regexps(values) - values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) - end - - def validate_string_or_regexp(value) - return false unless value.is_a?(String) - return validate_regexp(value) if look_like_regexp?(value) - - true - end - end - - class TypeValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - type = options[:with] - raise unless type.is_a?(Class) - - unless value.is_a?(type) - message = options[:message] || "should be a #{type.name}" - record.errors.add(attribute, message) - end - end - end - - class VariablesValidator < ActiveModel::EachValidator - include LegacyValidationHelpers - - def validate_each(record, attribute, value) - unless validate_variables(value) - record.errors.add(attribute, 'should be a hash of key value pairs') - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 8acab605c91..c9d0c7cb568 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,14 +7,14 @@ module Gitlab ## # Entry that represents environment variables. # - class Variables < Node - include Validatable + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable validations do validates :config, variables: true end - def self.default + def self.default(**) {} end diff --git a/lib/gitlab/ci/config/extendable.rb b/lib/gitlab/ci/config/extendable.rb new file mode 100644 index 00000000000..a43901c69fe --- /dev/null +++ b/lib/gitlab/ci/config/extendable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + include Enumerable + + ExtensionError = Class.new(StandardError) + + def initialize(hash) + @hash = hash.to_h.deep_dup + + each { |entry| entry.extend! if entry.extensible? } + end + + def each + @hash.each_key do |key| + yield Extendable::Entry.new(key, @hash) + end + end + + def to_hash + @hash.to_h + end + end + end + end +end diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb new file mode 100644 index 00000000000..7793db09d33 --- /dev/null +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Extendable + class Entry + InvalidExtensionError = Class.new(Extendable::ExtensionError) + CircularDependencyError = Class.new(Extendable::ExtensionError) + NestingTooDeepError = Class.new(Extendable::ExtensionError) + + MAX_NESTING_LEVELS = 10 + + attr_reader :key + + def initialize(key, context, parent = nil) + @key = key + @context = context + @parent = parent + + unless @context.key?(@key) + raise StandardError, 'Invalid entry key!' + end + end + + def extensible? + value.is_a?(Hash) && value.key?(:extends) + end + + def value + @value ||= @context.fetch(@key) + end + + def base_hash! + @base ||= Extendable::Entry + .new(extends_key, @context, self) + .extend! + end + + def extends_key + value.fetch(:extends).to_s.to_sym if extensible? + end + + def ancestors + @ancestors ||= Array(@parent&.ancestors) + Array(@parent&.key) + end + + def extend! + return value unless extensible? + + if unknown_extension? + raise Entry::InvalidExtensionError, + "#{key}: unknown key in `extends`" + end + + if invalid_base? + raise Entry::InvalidExtensionError, + "#{key}: invalid base hash in `extends`" + end + + if nesting_too_deep? + raise Entry::NestingTooDeepError, + "#{key}: nesting too deep in `extends`" + end + + if circular_dependency? + raise Entry::CircularDependencyError, + "#{key}: circular dependency detected in `extends`" + end + + @context[key] = base_hash!.deep_merge(value) + end + + private + + def nesting_too_deep? + ancestors.count > MAX_NESTING_LEVELS + end + + def circular_dependency? + ancestors.include?(key) + end + + def unknown_extension? + !@context.key?(extends_key) + end + + def invalid_base? + !@context[extends_key].is_a?(Hash) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb new file mode 100644 index 00000000000..a747886093c --- /dev/null +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Base + include Gitlab::Utils::StrongMemoize + + attr_reader :location, :params, :context, :errors + + YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze + + Context = Struct.new(:project, :sha, :user) + + def initialize(params, context) + @params = params + @context = context + @errors = [] + + validate! + end + + def matching? + location.present? + end + + def invalid_extension? + location.nil? || !::File.basename(location).match?(YAML_WHITELIST_EXTENSION) + end + + def valid? + errors.none? + end + + def error_message + errors.first + end + + def content + raise NotImplementedError, 'subclass must implement fetching raw content' + end + + def to_hash + @hash ||= Gitlab::Config::Loader::Yaml.new(content).load! + rescue Gitlab::Config::Loader::FormatError + nil + end + + protected + + def validate! + validate_location! + validate_content! if errors.none? + validate_hash! if errors.none? + end + + def validate_location! + if invalid_extension? + errors.push("Included file `#{location}` does not have YAML extension!") + end + end + + def validate_content! + if content.blank? + errors.push("Included file `#{location}` is empty or does not exist!") + end + end + + def validate_hash! + if to_hash.blank? + errors.push("Included file `#{location}` does not have valid YAML syntax!") + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb new file mode 100644 index 00000000000..2535d178ba8 --- /dev/null +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Local < Base + include Gitlab::Utils::StrongMemoize + + def initialize(params, context) + @location = params[:local] + + super + end + + def content + strong_memoize(:content) { fetch_local_content } + end + + private + + def validate_content! + if content.nil? + errors.push("Local file `#{location}` does not exist!") + elsif content.blank? + errors.push("Local file `#{location}` is empty!") + end + end + + def fetch_local_content + context.project.repository.blob_data_at(context.sha, location) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb new file mode 100644 index 00000000000..e75540dbe5a --- /dev/null +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Project < Base + include Gitlab::Utils::StrongMemoize + + attr_reader :project_name, :ref_name + + def initialize(params, context = {}) + @location = params[:file] + @project_name = params[:project] + @ref_name = params[:ref] || 'HEAD' + + super + end + + def matching? + super && project_name.present? + end + + def content + strong_memoize(:content) { fetch_local_content } + end + + private + + def validate_content! + if !can_access_local_content? + errors.push("Project `#{project_name}` not found or access denied!") + elsif sha.nil? + errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!") + elsif content.nil? + errors.push("Project `#{project_name}` file `#{location}` does not exist!") + elsif content.blank? + errors.push("Project `#{project_name}` file `#{location}` is empty!") + end + end + + def project + strong_memoize(:project) do + ::Project.find_by_full_path(project_name) + end + end + + def can_access_local_content? + Ability.allowed?(context.user, :download_code, project) + end + + def fetch_local_content + return unless can_access_local_content? + return unless sha + + project.repository.blob_data_at(sha, location) + rescue GRPC::NotFound, GRPC::Internal + nil + end + + def sha + strong_memoize(:sha) do + project.commit(ref_name).try(:sha) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb new file mode 100644 index 00000000000..567a86c47e5 --- /dev/null +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Remote < Base + include Gitlab::Utils::StrongMemoize + + def initialize(params, context) + @location = params[:remote] + + super + end + + def content + strong_memoize(:content) { fetch_remote_content } + end + + private + + def validate_location! + super + + unless ::Gitlab::UrlSanitizer.valid?(location) + errors.push("Remote file `#{location}` does not have a valid address!") + end + end + + def fetch_remote_content + begin + response = Gitlab::HTTP.get(location) + rescue SocketError + errors.push("Remote file `#{location}` could not be fetched because of a socket error!") + rescue Timeout::Error + errors.push("Remote file `#{location}` could not be fetched because of a timeout error!") + rescue Gitlab::HTTP::Error + errors.push("Remote file `#{location}` could not be fetched because of HTTP error!") + rescue Gitlab::HTTP::BlockedUrlError => e + errors.push("Remote file could not be fetched because #{e}!") + end + + if response&.code.to_i >= 400 + errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!") + end + + response.to_s if errors.none? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb new file mode 100644 index 00000000000..54f4cf74c4d --- /dev/null +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + module File + class Template < Base + attr_reader :location, :project + + SUFFIX = '.gitlab-ci.yml'.freeze + + def initialize(params, context) + @location = params[:template] + + super + end + + def content + strong_memoize(:content) { fetch_template_content } + end + + private + + def validate_location! + super + + unless template_name_valid? + errors.push("Template file `#{location}` is not a valid location!") + end + end + + def template_name + return unless template_name_valid? + + location.first(-SUFFIX.length) + end + + def template_name_valid? + location.to_s.end_with?(SUFFIX) + end + + def fetch_template_content + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb new file mode 100644 index 00000000000..108bfd5eb43 --- /dev/null +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Mapper + include Gitlab::Utils::StrongMemoize + + FILE_CLASSES = [ + External::File::Remote, + External::File::Template, + External::File::Local, + External::File::Project + ].freeze + + AmbigiousSpecificationError = Class.new(StandardError) + + def initialize(values, project:, sha:, user:) + @locations = Array.wrap(values.fetch(:include, [])) + @project = project + @sha = sha + @user = user + end + + def process + locations + .compact + .map(&method(:normalize_location)) + .map(&method(:select_first_matching)) + end + + private + + attr_reader :locations, :project, :sha, :user + + # convert location if String to canonical form + def normalize_location(location) + if location.is_a?(String) + normalize_location_string(location) + else + location.deep_symbolize_keys + end + end + + def normalize_location_string(location) + if ::Gitlab::UrlSanitizer.valid?(location) + { remote: location } + else + { local: location } + end + end + + def select_first_matching(location) + matching = FILE_CLASSES.map do |file_class| + file_class.new(location, context) + end.select(&:matching?) + + raise AmbigiousSpecificationError, "Include `#{location.to_json}` needs to match exactly one accessor!" unless matching.one? + + matching.first + end + + def context + strong_memoize(:context) do + External::File::Base::Context.new(project, sha, user) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb new file mode 100644 index 00000000000..69bc164a039 --- /dev/null +++ b/lib/gitlab/ci/config/external/processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Processor + IncludeError = Class.new(StandardError) + + def initialize(values, project:, sha:, user:) + @values = values + @external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process + @content = {} + rescue External::Mapper::AmbigiousSpecificationError => e + raise IncludeError, e.message + end + + def perform + return @values if @external_files.empty? + + validate_external_files! + merge_external_files! + append_inline_content! + remove_include_keyword! + end + + private + + def validate_external_files! + @external_files.each do |file| + raise IncludeError, file.error_message unless file.valid? + end + end + + def merge_external_files! + @external_files.each do |file| + @content.deep_merge!(file.to_hash) + end + end + + def append_inline_content! + @content.deep_merge!(@values) + end + + def remove_include_keyword! + @content.tap { @content.delete(:include) } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb new file mode 100644 index 00000000000..191f5d09645 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + def initialize(jobs_config) + @jobs_config = jobs_config + end + + def normalize_jobs + extract_parallelized_jobs! + return @jobs_config if @parallelized_jobs.empty? + + parallelized_config = parallelize_jobs + parallelize_dependencies(parallelized_config) + end + + private + + def extract_parallelized_jobs! + @parallelized_jobs = {} + + @jobs_config.each do |job_name, config| + if config[:parallel] + @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end + end + + @parallelized_jobs + end + + def parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if @parallelized_jobs.key?(job_name) + @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } + else + hash[job_name] = config + end + + hash + end + end + + def parallelize_dependencies(parallelized_config) + parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) + parallelized_config.each_with_object({}) do |(job_name, config), hash| + if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? + parallelized_deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + deps = config[:dependencies] - intersection + parallelized_deps + hash[job_name] = config.merge(dependencies: deps) + else + hash[job_name] = config + end + + hash + end + end + + def self.parallelize_job_names(name, total) + Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + end + end + end + end +end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 73f36735e35..94f4a4e36c9 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class CronParser @@ -33,7 +35,7 @@ module Gitlab # NOTE: # cron_timezone can only accept timezones listed in TZInfo::Timezone. # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, - # because Rufus::Scheduler only supports TZInfo::Timezone. + # because Fugit::Cron only supports TZInfo::Timezone. # # For example, those codes have the same effect. # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) @@ -45,10 +47,7 @@ module Gitlab # If you want to know more, please take a look # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb def try_parse_cron(cron, cron_timezone) - cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") - cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) - rescue - # noop + Fugit::Cron.parse("#{cron} #{cron_timezone}") end end end diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index 0daddaa638c..58d55b1bd6f 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -1,9 +1,13 @@ +# frozen_string_literal: true + module Gitlab module Ci::MaskSecret class << self def mask!(value, token) return value unless value.present? && token.present? + # We assume 'value' must be mutable, given + # that frozen string is enabled. value.gsub!(token, 'x' * token.length) value end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb index 3994a50772b..fbdb84c0522 100644 --- a/lib/gitlab/ci/model.rb +++ b/lib/gitlab/ci/model.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Model diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb new file mode 100644 index 00000000000..eb63e6c8363 --- /dev/null +++ b/lib/gitlab/ci/parsers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + ParserNotFoundError = Class.new(ParserError) + + def self.parsers + { + junit: ::Gitlab::Ci::Parsers::Test::Junit + } + end + + def self.fabricate!(file_type) + parsers.fetch(file_type.to_sym).new + rescue KeyError + raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" + end + end + end +end diff --git a/lib/gitlab/ci/parsers/parser_error.rb b/lib/gitlab/ci/parsers/parser_error.rb new file mode 100644 index 00000000000..ef327737cdb --- /dev/null +++ b/lib/gitlab/ci/parsers/parser_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + ParserError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb new file mode 100644 index 00000000000..dca60eabc1c --- /dev/null +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Test + class Junit + JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(xml_data, test_suite) + root = Hash.from_xml(xml_data) + + all_cases(root) do |test_case| + test_case = create_test_case(test_case) + test_suite.add_test_case(test_case) + end + rescue Nokogiri::XML::SyntaxError + raise JunitParserError, "XML parsing failed" + rescue + raise JunitParserError, "JUnit parsing failed" + end + + private + + def all_cases(root, parent = nil, &blk) + return unless root.present? + + [root].flatten.compact.map do |node| + next unless node.is_a?(Hash) + + # we allow only one top-level 'testsuites' + all_cases(node['testsuites'], root, &blk) unless parent + + # we require at least one level of testsuites or testsuite + each_case(node['testcase'], &blk) if parent + + # we allow multiple nested 'testsuite' (eg. PHPUnit) + all_cases(node['testsuite'], root, &blk) + end + end + + def each_case(testcase, &blk) + return unless testcase.present? + + [testcase].flatten.compact.map(&blk) + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index efed19da21c..bab1c73e2f1 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index b5eb0cfa2f0..41632211374 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -14,7 +16,7 @@ module Gitlab trigger_requests: Array(@command.trigger_request), user: @command.current_user, pipeline_schedule: @command.schedule, - protected: @command.protected_ref?, + merge_request: @command.merge_request, variables_attributes: Array(@command.variables_attributes) ) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index a53c80d34f7..e62d547d862 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,13 +1,16 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module Ci module Pipeline module Chain Command = Struct.new( :source, :project, :current_user, :origin_ref, :checkout_sha, :after_sha, :before_sha, - :trigger_request, :schedule, + :trigger_request, :schedule, :merge_request, :ignore_skip_ci, :save_incompleted, - :seeds_block, :variables_attributes + :seeds_block, :variables_attributes, :push_options ) do include Gitlab::Utils::StrongMemoize @@ -51,7 +54,13 @@ module Gitlab # rubocop:disable Naming/FileName def protected_ref? strong_memoize(:protected_ref) do - project.protected_for?(ref) + project.protected_for?(origin_ref) + end + end + + def ambiguous_ref? + strong_memoize(:ambiguous_ref) do + project.repository.ambiguous_ref?(origin_ref) end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index f4c8d5342c1..aa627bdb009 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -6,20 +8,7 @@ module Gitlab include Chain::Helpers def perform! - ::Ci::Pipeline.transaction do - pipeline.save! - - ## - # Create environments before the pipeline starts. - # - pipeline.builds.each do |build| - if build.has_environment? - project.environments.find_or_create_by( - name: build.expanded_environment_name - ) - end - end - end + pipeline.save! rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index bf1380a1da9..6bb3a75291b 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 69b8a8fc68f..0405292a25b 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -8,6 +10,13 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + # Protect the pipeline. This is assigned in Populate instead of + # Build to prevent erroring out on ambiguous refs. + pipeline.protected = @command.protected_ref? + ## # Populate pipeline with block argument of CreatePipelineService#execute. # diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index e24630656d3..99780409085 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 32cbb7ca6af..79bbcc1ed1e 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -6,6 +8,7 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PUSH_OPTION = 'ci.skip' def perform! if skipped? @@ -14,7 +17,7 @@ module Gitlab end def skipped? - !@command.ignore_skip_ci && commit_message_skips_ci? + !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) end def break? @@ -30,6 +33,10 @@ module Gitlab !!(@pipeline.git_commit_message =~ SKIP_PATTERN) end end + + def push_option_skips_ci? + !!(@command.push_options&.include?(SKIP_PUSH_OPTION)) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 13c6fedd831..ebd7e6e8289 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index a3bd2a5a23a..28c38cc3d18 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 9699c24e5b6..9c6c2bc8e25 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -14,6 +16,10 @@ module Gitlab unless @command.sha return error('Commit not found') end + + if @command.ambiguous_ref? + return error('Ref is ambiguous') + end end def break? diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index 469fc094cc8..de24bbf688b 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -86,6 +88,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) status = %w[success failed running canceled] builds = pipeline.builds.latest @@ -93,6 +96,7 @@ module Gitlab from_builds(builds) end + # rubocop: enable CodeReuse/ActiveRecord def from_builds(builds) now = Time.now @@ -134,9 +138,11 @@ module Gitlab Period.new(previous.first, [previous.last, current.last].max) end + # rubocop: disable CodeReuse/ActiveRecord def process_duration(periods) periods.sum(&:duration) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb index f57df7c5637..61d392121d8 100644 --- a/lib/gitlab/ci/pipeline/expression.rb +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 047ab66e9b3..70c774416f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 3a2f0c6924e..668e85f5b9e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 10957598f76..cd17bc4d78b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index a2778716924..be7258c201a 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index f640d0b5855..3ebceb92eb7 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 9b239c29ea4..d7e6dacf068 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 346c92dc51e..2db2bf011f1 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index f2611d65faf..ef9ddb6cae9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 37643c8ef53..85c0899e4f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 4cacb1e62c9..f26542361a2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 90f94d0b763..ed184309ab4 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index b36f1e0f865..b03611f756e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb index 58211800b88..513d43f6fca 100644 --- a/lib/gitlab/ci/pipeline/expression/token.rb +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb index e7a2e5511cf..db0a1ea4dab 100644 --- a/lib/gitlab/ci/pipeline/preloader.rb +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -5,23 +5,47 @@ module Gitlab module Pipeline # Class for preloading data associated with pipelines such as commit # authors. - module Preloader - def self.preload(pipelines) - # This ensures that all the pipeline commits are eager loaded before we - # start using them. + class Preloader + def self.preload!(pipelines) + ## + # This preloads all commits at once, because `Ci::Pipeline#commit` is + # using a lazy batch loading, what results in only one batched Gitaly + # call. + # pipelines.each(&:commit) pipelines.each do |pipeline| - # This preloads the author of every commit. We're using "lazy_author" - # here since "author" immediately loads the data on the first call. - pipeline.commit.try(:lazy_author) - - # This preloads the number of warnings for every pipeline, ensuring - # that Ci::Pipeline#has_warnings? doesn't execute any additional - # queries. - pipeline.number_of_warnings + self.new(pipeline).tap do |preloader| + preloader.preload_commit_authors + preloader.preload_pipeline_warnings + preloader.preload_stages_warnings + end end end + + def initialize(pipeline) + @pipeline = pipeline + end + + def preload_commit_authors + # This also preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. + @pipeline.commit.try(:lazy_author) + end + + def preload_pipeline_warnings + # This preloads the number of warnings for every pipeline, ensuring + # that Ci::Pipeline#has_warnings? doesn't execute any additional + # queries. + @pipeline.number_of_warnings + end + + def preload_stages_warnings + # This preloads the number of warnings for every stage, ensuring + # that Ci::Stage#has_warnings? doesn't execute any additional + # queries. + @pipeline.stages.each { |stage| stage.number_of_warnings } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index db9706924bb..1fd3a61017f 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 6980b0b7aff..d8296940a04 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -36,9 +38,17 @@ module Gitlab ) end + def bridge? + @attributes.to_h.dig(:options, :trigger).present? + end + def to_resource strong_memoize(:resource) do - ::Ci::Build.new(attributes) + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 2b58d9863a0..9c15064756a 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -37,7 +39,13 @@ module Gitlab def to_resource strong_memoize(:stage) do ::Ci::Stage.new(attributes).tap do |stage| - seeds.each { |seed| stage.builds << seed.to_resource } + seeds.each do |seed| + if seed.bridge? + stage.bridges << seed.to_resource + else + stage.builds << seed.to_resource + end + end end end end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb new file mode 100644 index 00000000000..292e273a03a --- /dev/null +++ b/lib/gitlab/ci/reports/test_case.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestCase + STATUS_SUCCESS = 'success'.freeze + STATUS_FAILED = 'failed'.freeze + STATUS_SKIPPED = 'skipped'.freeze + STATUS_ERROR = 'error'.freeze + STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze + + attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key + + def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil) + @name = name + @classname = classname + @file = file + @execution_time = execution_time.to_f + @status = status + @system_output = system_output + @stack_trace = stack_trace + @key = sanitize_key_name("#{classname}_#{name}") + end + + private + + def sanitize_key_name(key) + key.gsub(/[^0-9A-Za-z]/, '-') + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb new file mode 100644 index 00000000000..7397ff35d46 --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestReports + attr_reader :test_suites + + def initialize + @test_suites = {} + end + + def get_suite(suite_name) + test_suites[suite_name] ||= TestSuite.new(suite_name) + end + + # rubocop: disable CodeReuse/ActiveRecord + def total_time + test_suites.values.sum(&:total_time) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def total_count + test_suites.values.sum(&:total_count) + end + # rubocop: enable CodeReuse/ActiveRecord + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}_count") do + # rubocop: disable CodeReuse/ActiveRecord + test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb new file mode 100644 index 00000000000..11810bdc0a8 --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_reports, :head_reports + + def initialize(base_reports, head_reports) + @base_reports = base_reports || TestReports.new + @head_reports = head_reports + end + + def suite_comparers + strong_memoize(:suite_comparers) do + head_reports.test_suites.map do |name, test_suite| + TestSuiteComparer.new(name, base_reports.get_suite(name), test_suite) + end + end + end + + def total_status + if suite_comparers.any? { |suite| suite.total_status == TestCase::STATUS_FAILED } + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + %w(total_count resolved_count failed_count).each do |method| + define_method(method) do + # rubocop: disable CodeReuse/ActiveRecord + suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb new file mode 100644 index 00000000000..b0391160c15 --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestSuite + attr_reader :name + attr_reader :test_cases + attr_reader :total_time + + def initialize(name = nil) + @name = name + @test_cases = {} + @total_time = 0.0 + @duplicate_cases = [] + end + + def add_test_case(test_case) + @duplicate_cases << test_case if existing_key?(test_case) + + @test_cases[test_case.status] ||= {} + @test_cases[test_case.status][test_case.key] = test_case + @total_time += test_case.execution_time + end + + # rubocop: disable CodeReuse/ActiveRecord + def total_count + test_cases.values.sum(&:count) + end + # rubocop: enable CodeReuse/ActiveRecord + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}") do + test_cases[status_type] || {} + end + + define_method("#{status_type}_count") do + test_cases[status_type]&.length.to_i + end + end + + private + + def existing_key?(test_case) + @test_cases[test_case.status]&.key?(test_case.key) + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb new file mode 100644 index 00000000000..9cb7db5934c --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestSuiteComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :name, :base_suite, :head_suite + + def initialize(name, base_suite, head_suite) + @name = name + @base_suite = base_suite || TestSuite.new + @head_suite = head_suite + end + + def new_failures + strong_memoize(:new_failures) do + head_suite.failed.reject do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def existing_failures + strong_memoize(:existing_failures) do + head_suite.failed.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def resolved_failures + strong_memoize(:resolved_failures) do + head_suite.success.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def total_count + head_suite.total_count + end + + def total_status + head_suite.total_status + end + + def resolved_count + resolved_failures.count + end + + def failed_count + new_failures.count + existing_failures.count + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb new file mode 100644 index 00000000000..4746195c618 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + module Common + def label + subject.description + end + + def has_details? + false + end + + def has_action? + false + end + + def details_path + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb new file mode 100644 index 00000000000..910de865483 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Factory < Status::Factory + def self.common_helpers + Status::Bridge::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb index 6c9125647ad..45d9ba41e92 100644 --- a/lib/gitlab/ci/status/build/action.rb +++ b/lib/gitlab/ci/status/build/action.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 024047d4983..43fb5cdbbe6 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb index c83e2734a73..0518b9e673d 100644 --- a/lib/gitlab/ci/status/build/canceled.rb +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c1fc70ac266..6a75ec5c37f 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb index 5be8e9de425..780fea23123 100644 --- a/lib/gitlab/ci/status/build/created.rb +++ b/lib/gitlab/ci/status/build/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb index 495227c2ffb..d74cfc1ee77 100644 --- a/lib/gitlab/ci/status/build/erased.rb +++ b/lib/gitlab/ci/status/build/erased.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 2b26ebb45a1..6e4bfe23f2b 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -5,6 +7,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::Build::Erased, + Status::Build::Scheduled, Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, @@ -14,6 +17,7 @@ module Gitlab Status::Build::Retryable], [Status::Build::Failed], [Status::Build::FailedAllowed, + Status::Build::Unschedule, Status::Build::Play, Status::Build::Stop], [Status::Build::Action], diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 155f4fc1343..d40454df737 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -1,17 +1,25 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status module Build class Failed < Status::Extended REASONS = { - 'unknown_failure' => 'unknown failure', - 'script_failure' => 'script failure', - 'api_failure' => 'API failure', - 'stuck_or_timeout_failure' => 'stuck or timeout failure', - 'runner_system_failure' => 'runner system failure', - 'missing_dependency_failure' => 'missing dependency failure' + unknown_failure: 'unknown failure', + script_failure: 'script failure', + api_failure: 'API failure', + stuck_or_timeout_failure: 'stuck or timeout failure', + runner_system_failure: 'runner system failure', + missing_dependency_failure: 'missing dependency failure', + runner_unsupported: 'unsupported runner', + stale_schedule: 'stale schedule', + job_execution_timeout: 'job execution timeout', + archived_failure: 'archived failure' }.freeze + private_constant :REASONS + def status_tooltip base_message end @@ -24,6 +32,10 @@ module Gitlab build.failed? end + def self.reasons + REASONS + end + private def base_message @@ -31,7 +43,11 @@ module Gitlab end def description - "<br> (#{REASONS[subject.failure_reason]})" + "- (#{failure_reason_message})" + end + + def failure_reason_message + self.class.reasons.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index ca0046fb1f7..d7570fdd3e2 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 042da6392d3..d01b09f1398 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb index 9dd9a27ad57..95f668295dd 100644 --- a/lib/gitlab/ci/status/build/pending.rb +++ b/lib/gitlab/ci/status/build/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index a8b9ebf0803..c66b8ca5654 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb index 6e190e4ee3c..b489dc68733 100644 --- a/lib/gitlab/ci/status/build/retried.rb +++ b/lib/gitlab/ci/status/build/retried.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 5aeb8e51480..eb6b3f21604 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb new file mode 100644 index 00000000000..0a09dbe5f42 --- /dev/null +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class Scheduled < Status::Extended + def illustration + { + image: 'illustrations/illustrations_scheduled-job_countdown.svg', + size: 'svg-394', + title: _("This is a delayed job to run in %{remainingTime}"), + content: _("This job will automatically run after its timer finishes. " \ + "Often they are used for incremental roll-out deploys " \ + "to production environments. When unscheduled it converts " \ + "into a manual action.") + } + end + + def status_tooltip + "delayed manual action (%{remainingTime})" + end + + def self.matches?(build, user) + build.scheduled? && build.scheduled_at + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb index 3e678d0baee..4fe2f7b3114 100644 --- a/lib/gitlab/ci/status/build/skipped.rb +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index dea838bfa39..a620e7ad126 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb new file mode 100644 index 00000000000..9110839cb55 --- /dev/null +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class Unschedule < Status::Extended + def label + 'unschedule action' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'time-out' + end + + def action_title + 'Unschedule' + end + + def action_button_title + _('Unschedule job') + end + + def action_path + unschedule_project_job_path(subject.project, subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index e6195a60d4f..07f37732023 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 9d6a2f51c11..ea773ee9944 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 846f00b83dd..fface4bb97b 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index 1e8101f8949..b72a28ed0b6 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 9307545b5b1..cd772819293 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status module External module Common def label - subject.description + subject.description.presence || super end def has_details? diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb index 07b15bd8d97..91fafb940a8 100644 --- a/lib/gitlab/ci/status/external/factory.rb +++ b/lib/gitlab/ci/status/external/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 15836c699c7..3446644eff8 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 27ce85bd3ed..770ed7d4d5a 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb index cfd4329a923..0b5ea0712ca 100644 --- a/lib/gitlab/ci/status/group/common.rb +++ b/lib/gitlab/ci/status/group/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index d118116cfc3..ee785856fdd 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index fc387e2fd25..50c92add400 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 6780780db32..cea7e6ed938 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index bf7e484ee9b..ed13a439be0 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 61bb07beb0f..7b34a2ea858 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb new file mode 100644 index 00000000000..e61acdcd167 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/delayed.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Pipeline + class Delayed < Status::Extended + def text + s_('CiStatusText|delayed') + end + + def label + s_('CiStatusLabel|waiting for delayed job') + end + + def self.matches?(pipeline, user) + pipeline.scheduled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 17f9a75f436..5d1a8bbd924 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -5,6 +7,7 @@ module Gitlab class Factory < Status::Factory def self.extended_statuses [[Status::SuccessWarning, + Status::Pipeline::Delayed, Status::Pipeline::Blocked]] end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index ee13905e46d..ac7dd74cdce 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb new file mode 100644 index 00000000000..16ad1da89e3 --- /dev/null +++ b/lib/gitlab/ci/status/scheduled.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class Scheduled < Status::Core + def text + s_('CiStatusText|delayed') + end + + def label + s_('CiStatusLabel|delayed') + end + + def icon + 'status_scheduled' + end + + def favicon + 'favicon_status_scheduled' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 0dbdc4de426..aaec1e1d201 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index bc99d925347..f12daaa9676 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -8,7 +10,9 @@ module Gitlab end def details_path - project_pipeline_path(subject.project, subject.pipeline, anchor: subject.name) + project_pipeline_path(subject.pipeline.project, + subject.pipeline, + anchor: subject.name) end def has_action? diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 4c37f084d07..58f4642510b 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 731013ec017..020f2c5b89f 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 32b4cf43e48..6632cd9b143 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml new file mode 100644 index 00000000000..6e138639b71 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -0,0 +1,45 @@ +# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny +image: openjdk:8-jdk + +variables: + ANDROID_COMPILE_SDK: "28" + ANDROID_BUILD_TOOLS: "28.0.2" + ANDROID_SDK_TOOLS: "4333796" + +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - wget --quiet --output-document=android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_TOOLS}.zip + - unzip -d android-sdk-linux android-sdk.zip + - echo y | android-sdk-linux/tools/bin/sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" >/dev/null + - echo y | android-sdk-linux/tools/bin/sdkmanager "platform-tools" >/dev/null + - echo y | android-sdk-linux/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" >/dev/null + - export ANDROID_HOME=$PWD/android-sdk-linux + - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ + - chmod +x ./gradlew + # temporarily disable checking for EPIPE error and use yes to accept all licenses + - set +o pipefail + - yes | android-sdk-linux/tools/bin/sdkmanager --licenses + - set -o pipefail + +stages: + - build + - test + +lintDebug: + stage: build + script: + - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + +assembleDebug: + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +debugTests: + stage: test + script: + - ./gradlew -Pci --console=plain :app:testDebug diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml new file mode 100644 index 00000000000..75a5bf142d2 --- /dev/null +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -0,0 +1,954 @@ +# Auto DevOps +# This CI/CD configuration provides a standard pipeline for +# * building a Docker image (using a buildpack if necessary), +# * storing the image in the container registry, +# * running tests from a buildpack, +# * running code quality analysis, +# * creating a review app for each topic branch, +# * and continuous deployment to production +# +# Test jobs may be disabled by setting environment variables: +# * test: TEST_DISABLED +# * code_quality: CODE_QUALITY_DISABLED +# * license_management: LICENSE_MANAGEMENT_DISABLED +# * performance: PERFORMANCE_DISABLED +# * sast: SAST_DISABLED +# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED +# * container_scanning: CONTAINER_SCANNING_DISABLED +# * dast: DAST_DISABLED +# * review: REVIEW_DISABLED +# * stop_review: REVIEW_DISABLED +# +# In order to deploy, you must have a Kubernetes cluster configured either +# via a project integration, or via group/project variables. +# AUTO_DEVOPS_DOMAIN must also be set as a variable at the group or project +# level, or manually added below. +# +# Continuous deployment to production is enabled by default. +# If you want to deploy to staging first, set STAGING_ENABLED environment variable. +# If you want to enable incremental rollout, either manual or time based, +# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed". +# If you want to use canary deployments, set CANARY_ENABLED environment variable. +# +# If Auto DevOps fails to detect the proper buildpack, or if you want to +# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the +# repository URL of the buildpack. +# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142 +# If you need multiple buildpacks, add a file to your project called +# `.buildpacks` that contains the URLs, one on each line, in order. +# Note: Auto CI does not work with multiple buildpacks yet + +image: alpine:latest + +variables: + # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level. + # AUTO_DEVOPS_DOMAIN: domain.example.com + + POSTGRES_USER: user + POSTGRES_PASSWORD: testing-password + POSTGRES_ENABLED: "true" + POSTGRES_DB: $CI_ENVIRONMENT_SLUG + + KUBERNETES_VERSION: 1.11.6 + HELM_VERSION: 2.12.2 + + DOCKER_DRIVER: overlay2 + +stages: + - build + - test + - review + - dast + - staging + - canary + - production + - incremental rollout 10% + - incremental rollout 25% + - incremental rollout 50% + - incremental rollout 100% + - performance + - cleanup + +build: + stage: build + image: docker:stable-git + services: + - docker:stable-dind + script: + - setup_docker + - build + only: + - branches + +test: + services: + - postgres:latest + variables: + POSTGRES_DB: test + stage: test + image: gliderlabs/herokuish:latest + script: + - setup_test_db + - cp -R . /tmp/app + - /bin/herokuish buildpack test + only: + - branches + except: + variables: + - $TEST_DISABLED + +code_quality: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - code_quality + artifacts: + paths: [gl-code-quality-report.json] + only: + - branches + except: + variables: + - $CODE_QUALITY_DISABLED + +license_management: + stage: test + image: + name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + entrypoint: [""] + allow_failure: true + script: + - license_management + artifacts: + paths: [gl-license-management-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\blicense_management\b/ + except: + variables: + - $LICENSE_MANAGEMENT_DISABLED + +performance: + stage: performance + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - performance + artifacts: + paths: + - performance.json + - sitespeed-results/ + only: + refs: + - branches + kubernetes: active + except: + variables: + - $PERFORMANCE_DISABLED + +sast: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - sast + artifacts: + reports: + sast: gl-sast-report.json + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ + except: + variables: + - $SAST_DISABLED + +dependency_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - dependency_scanning + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + except: + variables: + - $DEPENDENCY_SCANNING_DISABLED + +container_scanning: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - setup_docker + - container_scanning + artifacts: + paths: [gl-container-scanning-report.json] + only: + refs: + - branches + variables: + - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + except: + variables: + - $CONTAINER_SCANNING_DISABLED + +dast: + stage: dast + allow_failure: true + image: registry.gitlab.com/gitlab-org/security-products/zaproxy + variables: + POSTGRES_DB: "false" + script: + - dast + artifacts: + paths: [gl-dast-report.json] + only: + refs: + - branches + kubernetes: active + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED + +review: + stage: review + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_SLUG.$AUTO_DEVOPS_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +stop_review: + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - install_dependencies + - initialize_tiller + - delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual + allow_failure: true + only: + refs: + - branches + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + stage: staging + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$AUTO_DEVOPS_DOMAIN + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + +# Canaries are also disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + stage: canary + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $CANARY_ENABLED + +.production: &production_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - delete canary + - delete rollout + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +production: + <<: *production_template + only: + refs: + - master + kubernetes: active + except: + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +production_manual: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy rollout $ROLLOUT_PERCENTAGE + - scale stable $((100-ROLLOUT_PERCENTAGE)) + - delete canary + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$AUTO_DEVOPS_DOMAIN + artifacts: + paths: [environment_url.txt] + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + when: manual + # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "manual" + - $INCREMENTAL_ROLLOUT_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + when: delayed + start_in: 5 minutes + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false + +# --------------------------------------------------------------------------- + +.auto_devops: &auto_devops | + # Auto DevOps variables and functions + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + export CI_APPLICATION_TAG=$CI_COMMIT_SHA + export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + + function registry_login() { + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi + } + + function container_scanning() { + registry_login + + docker run -d --name db arminc/clair-db:latest + docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.1 + apk add -U wget ca-certificates + docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} + wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + mv clair-scanner_linux_amd64 clair-scanner + chmod +x clair-scanner + touch clair-whitelist.yml + retries=0 + echo "Waiting for clair daemon to start" + while( ! wget -T 10 -q -O /dev/null http://docker:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done + ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + } + + function code_quality() { + docker run --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code + } + + function license_management() { + /run.sh analyze . + } + + function sast() { + case "$CI_SERVER_VERSION" in + *-ee) + + # Deprecation notice for CONFIDENCE_LEVEL variable + if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then + SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" + echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" + fi + + docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function dependency_scanning() { + case "$CI_SERVER_VERSION" in + *-ee) + docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + ;; + *) + echo "GitLab EE is required" + ;; + esac + } + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + # Extracts variables prefixed with K8S_SECRET_ + # and creates a Kubernetes secret. + # + # e.g. If we have the following environment variables: + # K8S_SECRET_A=value1 + # K8S_SECRET_B=multi\ word\ value + # + # Then we will create a secret with the following key-value pairs: + # data: + # A: dmFsdWUxCg== + # B: bXVsdGkgd29yZCB2YWx1ZQo= + function create_application_secret() { + track="${1-stable}" + export APPLICATION_SECRET_NAME=$(application_secret_name "$track") + + env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables + + kubectl create secret \ + -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ + --from-env-file k8s_prefixed_variables -o yaml --dry-run | + kubectl replace -n "$KUBE_NAMESPACE" --force -f - + + export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) + + rm k8s_prefixed_variables + } + + function deploy_name() { + name="$CI_ENVIRONMENT_SLUG" + track="${1-stable}" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo $name + } + + function application_secret_name() { + track="${1-stable}" + name=$(deploy_name "$track") + + echo "${name}-secret" + } + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name=$(deploy_name "$track") + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + create_application_secret "$track" + + env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') + eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS + if [ -n "$env_ADDITIONAL_HOSTS" ]; then + additional_hosts="{$env_ADDITIONAL_HOSTS}" + elif [ -n "$ADDITIONAL_HOSTS" ]; then + additional_hosts="{$ADDITIONAL_HOSTS}" + fi + + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + echo "Deploying first release with database initialization..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + echo "Deploying second release..." + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + echo "Deploying new release..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$CI_APPLICATION_REPOSITORY" \ + --set image.tag="$CI_APPLICATION_TAG" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$AUTO_DEVOPS_DOMAIN" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "deployment/$name" + } + + function scale() { + track="${1-stable}" + percentage="${2-100}" + name=$(deploy_name "$track") + + replicas=$(get_replicas "$track" "$percentage") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm upgrade --reuse-values \ + --wait \ + --set replicaCount="$replicas" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + } + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + curl -L -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add glibc-2.28-r0.apk + rm glibc-2.28-r0.apk + + curl "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ + helm version --client + tiller -version + + curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function setup_docker() { + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + } + + function setup_test_db() { + if [ -z ${KUBERNETES_PORT+x} ]; then + DB_HOST=postgres + else + DB_HOST=localhost + fi + export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + auto_chart_name=${auto_chart_name%.tar.gz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + function check_kube_domain() { + if [ -z ${AUTO_DEVOPS_DOMAIN+x} ]; then + echo "In order to deploy or use Review Apps, AUTO_DEVOPS_DOMAIN variable must be set" + echo "You can do it in Auto DevOps project settings or defining a variable at group or project level" + echo "You can also manually add it in .gitlab-ci.yml" + false + else + true + fi + } + + function build() { + registry_login + + if [[ -f Dockerfile ]]; then + echo "Building Dockerfile-based application..." + docker build \ + --build-arg HTTP_PROXY="$HTTP_PROXY" \ + --build-arg http_proxy="$http_proxy" \ + --build-arg HTTPS_PROXY="$HTTPS_PROXY" \ + --build-arg https_proxy="$https_proxy" \ + --build-arg FTP_PROXY="$FTP_PROXY" \ + --build-arg ftp_proxy="$ftp_proxy" \ + --build-arg NO_PROXY="$NO_PROXY" \ + --build-arg no_proxy="$no_proxy" \ + -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" . + else + echo "Building Heroku-based application using gliderlabs/herokuish docker image..." + docker run -i \ + -e BUILDPACK_URL \ + -e HTTP_PROXY \ + -e http_proxy \ + -e HTTPS_PROXY \ + -e https_proxy \ + -e FTP_PROXY \ + -e ftp_proxy \ + -e NO_PROXY \ + -e no_proxy \ + --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" gliderlabs/herokuish /bin/herokuish buildpack build + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + + echo "Configuring $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG docker image..." + docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" /bin/herokuish procfile start web + docker commit "$CI_CONTAINER_NAME" "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + docker rm "$CI_CONTAINER_NAME" >/dev/null + echo "" + fi + + echo "Pushing to GitLab Container Registry..." + docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" + echo "" + } + + function initialize_tiller() { + echo "Checking Tiller..." + + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & + echo "Tiller is listening on ${HELM_HOST}" + + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ + --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function dast() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir /zap/wrk/ + /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true + cp /zap/wrk/gl-dast-report.json . + } + + function performance() { + export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + + mkdir gitlab-exporter + wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + + mkdir sitespeed-results + + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + else + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + fi + + mv sitespeed-results/data/performance.json performance.json + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + + function delete() { + track="${1-stable}" + name=$(deploy_name "$track") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm delete --purge "$name" + fi + + secret_name=$(application_secret_name "$track") + kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" + } + +before_script: + - *auto_devops diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml new file mode 100644 index 00000000000..2d218b2e164 --- /dev/null +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -0,0 +1,35 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# you can delete this line if you're not using Docker +image: busybox:latest + +before_script: + - echo "Before script section" + - echo "For example you might run an update here or install a build dependency" + - echo "Or perhaps you might print out some debugging details" + +after_script: + - echo "After script section" + - echo "For example you might do some cleanup here" + +build1: + stage: build + script: + - echo "Do your build here" + +test1: + stage: test + script: + - echo "Do a test here" + - echo "For example run a test suite" + +test2: + stage: test + script: + - echo "Do another parallel test here" + - echo "For example run a lint test" + +deploy1: + stage: deploy + script: + - echo "Do your deploy here" diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml new file mode 100644 index 00000000000..c83c49d8c95 --- /dev/null +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -0,0 +1,26 @@ +# use the official gcc image, based on debian +# can use verions as well, like gcc:5.2 +# see https://hub.docker.com/_/gcc/ +image: gcc + +build: + stage: build + # instead of calling g++ directly you can also use some build toolkit like make + # install the necessary build tools when needed + # before_script: + # - apt update && apt -y install make autoconf + script: + - g++ helloworld.cpp -o mybinary + artifacts: + paths: + - mybinary + # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time + # cache: + # paths: + # - "*.o" + +# run tests using the binary built before +test: + stage: test + script: + - ./runmytests.sh diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml new file mode 100644 index 00000000000..4d5b6484d6e --- /dev/null +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -0,0 +1,51 @@ +# This file uses Test Kitchen with the kitchen-dokken driver to +# perform functional testing. Doing so requires that your runner be a +# Docker runner configured for privileged mode. Please see +# https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode +# for help configuring your runner properly, or, if you want to switch +# to a different driver, see http://kitchen.ci/docs/drivers + +image: "chef/chefdk" +services: + - docker:dind + +variables: + DOCKER_HOST: "tcp://docker:2375" + KITCHEN_LOCAL_YAML: ".kitchen.dokken.yml" + +stages: + - lint + - unit + - functional + +foodcritic: + stage: lint + script: + - chef exec foodcritic . + +cookstyle: + stage: lint + script: + - chef exec cookstyle . + +chefspec: + stage: unit + script: + - chef exec rspec spec + +# Set up your test matrix here. Example: +#verify-centos-6: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-6 --destroy=always +# +#verify-centos-7: +# stage: functional +# before_script: +# - apt-get update +# - apt-get -y install rsync +# script: +# - kitchen verify default-centos-7 --destroy=always diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml new file mode 100644 index 00000000000..f066285b1ad --- /dev/null +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Based on openjdk:8, already includes lein +image: clojure:lein-2.7.0 +# If you need to configure a database, add a `services` section here +# See https://docs.gitlab.com/ce/ci/services/postgres.html +# Make sure you configure the connection as well + +before_script: + # If you need to install any external applications, like a + # postgres client, you may want to uncomment the line below: + # + #- apt-get update -y + # + # Retrieve project dependencies + # Do this on before_script since it'll be shared between both test and + # any production sections a user adds + - lein deps + +test: + script: + # If you need to run any migrations or configure the database, this + # would be the point to do it. + - lein test diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml new file mode 100644 index 00000000000..36386a19fdc --- /dev/null +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/crystallang/crystal/ +image: "crystallang/crystal:latest" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# variables: +# POSTGRES_DB: database_name + +# Cache shards in between builds +cache: + paths: + - lib + +# This is a basic example for a shard or script which doesn't use +# services such as redis or postgres +before_script: + - apt-get update -qq && apt-get install -y -qq libxml2-dev + - crystal -v # Print out Crystal version for debugging + - shards + +# If you are using built-in Crystal Spec. +spec: + script: + - crystal spec + +# If you are using minitest.cr +minitest: + script: + - crystal test/spec_test.cr # change to the file(s) you execute for tests diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml new file mode 100644 index 00000000000..57afcbbe8b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -0,0 +1,49 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python +image: python:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - ~/.cache/pip/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - python -V # Print out python version for debugging + # Uncomment next line if your Django app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - pip install -r requirements.txt + +# To get Django tests to work you may need to create a settings file using +# the following DATABASES: +# +# DATABASES = { +# 'default': { +# 'ENGINE': 'django.db.backends.postgresql_psycopg2', +# 'NAME': 'ci', +# 'USER': 'postgres', +# 'PASSWORD': 'postgres', +# 'HOST': 'postgres', +# 'PORT': '5432', +# }, +# } +# +# and then adding `--settings app.settings.ci` (or similar) to the test command + +test: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - python manage.py test diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml new file mode 100644 index 00000000000..eeefadaa019 --- /dev/null +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Official docker image. +image: docker:latest + +services: + - docker:dind + +before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +build-master: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE" . + - docker push "$CI_REGISTRY_IMAGE" + only: + - master + +build: + stage: build + script: + - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . + - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" + except: + - master diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml new file mode 100644 index 00000000000..cf9c731637c --- /dev/null +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -0,0 +1,18 @@ +image: elixir:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +before_script: + - mix local.rebar --force + - mix local.hex --force + - mix deps.get + +mix: + script: + - mix test diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml new file mode 100644 index 00000000000..d572d7a1edc --- /dev/null +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -0,0 +1,35 @@ +image: golang:latest + +variables: + # Please edit to your GitLab project + REPO_NAME: gitlab.com/namespace/project + +# The problem is that to be able to use go get, one needs to put +# the repository in the $GOPATH. So for example if your gitlab domain +# is gitlab.com, and that your repository is namespace/project, and +# the default GOPATH being /go, then you'd need to have your +# repository in /go/src/gitlab.com/namespace/project +# Thus, making a symbolic link corrects this. +before_script: + - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) + - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME + - cd $GOPATH/src/$REPO_NAME + +stages: + - test + - build + +format: + stage: test + script: + - go fmt $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - go test -race $(go list ./... | grep -v /vendor/) + +compile: + stage: build + script: + - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + artifacts: + paths: + - mybinary diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml new file mode 100644 index 00000000000..48d98dddfad --- /dev/null +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -0,0 +1,36 @@ +# This is the Gradle build system for JVM applications +# https://gradle.org/ +# https://github.com/gradle/gradle +image: gradle:alpine + +# Disable the Gradle daemon for Continuous Integration servers as correctness +# is usually a priority over speed in CI environments. Using a fresh +# runtime for each build is more reliable since the runtime is completely +# isolated from any previous builds. +variables: + GRADLE_OPTS: "-Dorg.gradle.daemon=false" + +before_script: + - export GRADLE_USER_HOME=`pwd`/.gradle + +build: + stage: build + script: gradle --build-cache assemble + cache: + key: "$CI_COMMIT_REF_NAME" + policy: push + paths: + - build + - .gradle + + +test: + stage: test + script: gradle check + cache: + key: "$CI_COMMIT_REF_NAME" + policy: pull + paths: + - build + - .gradle + diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml new file mode 100644 index 00000000000..7fc698d50cf --- /dev/null +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -0,0 +1,40 @@ +# This template uses the java:8 docker image because there isn't any +# official Grails image at this moment +# +# Grails Framework https://grails.org/ is a powerful Groovy-based web application framework for the JVM +# +# This yml works with Grails 3.x only +# Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) +# Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) +# If you use Angular profile, this yml it's prepared to work with it + +image: java:8 + +variables: + GRAILS_VERSION: "3.1.9" + GRADLE_VERSION: "2.13" + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install gradle $GRADLE_VERSION < /dev/null + - sdk use gradle $GRADLE_VERSION +# As it's not a good idea to version gradle.properties feel free to add your +# environments variable here + - echo grailsVersion=$GRAILS_VERSION > gradle.properties + - echo gradleWrapperVersion=2.14 >> gradle.properties +# refresh dependencies from your project + - ./gradlew --refresh-dependencies +# Be aware that if you are using Angular profile, +# Bower cannot be run as root if you don't allow it before. +# Feel free to remove next line if you are not using Bower + - echo {\"allow_root\":true} > /root/.bowerrc + +# This build job does the full grails pipeline +# (compile, test, integrationTest, war, assemble). +build: + script: + - ./gradlew build
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml new file mode 100644 index 00000000000..04c21b4725d --- /dev/null +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -0,0 +1,76 @@ +# This is an example .gitlab-ci.yml file to test (and optionally report the coverage +# results of) your [Julia][1] packages. Please refer to the [documentation][2] +# for more information about package development in Julia. +# +# Here, it is assumed that your Julia package is named `MyPackage`. Change it to +# whatever name you have given to your package. +# +# [1]: http://julialang.org/ +# [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html + +# Below is the template to run your tests in Julia +.test_template: &test_definition + # Uncomment below if you would like to run the tests on specific references + # only, such as the branches `master`, `development`, etc. + # only: + # - master + # - development + script: + # Let's run the tests. Substitute `coverage = false` below, if you do not + # want coverage results. + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage"); Pkg.test("MyPackage"; coverage = true)' + # Comment out below if you do not want coverage results. + - julia -e 'using Pkg; Pkg.add("Coverage"); + import MyPackage; cd(joinpath(dirname(pathof(MyPackage)), "..")); + using Coverage; cl, tl = get_summary(process_folder()); + println("(", cl/tl*100, "%) covered")' + +# Name a test and select an appropriate image. +# images comes from Docker hub +test:0.7: + image: julia:0.7 + <<: *test_definition + +test:1.0: + image: julia:1.0 + <<: *test_definition + +# Maybe you would like to test your package against the development branch: +# test:1.1-dev (not sure there is such an image in docker, so not tested yet): +# image: julia:v1.1-dev +# # ... allowing for failures, since we are testing against the development +# # branch: +# allow_failure: true +# <<: *test_definition + +# REMARK: Do not forget to enable the coverage feature for your project, if you +# are using code coverage reporting above. This can be done by +# +# - Navigating to the `CI/CD Pipelines` settings of your project, +# - Copying and pasting the default `Simplecov` regex example provided, i.e., +# `\(\d+.\d+\%\) covered` in the `test coverage parsing` textfield. + +# Example documentation deployment +pages: + image: julia:0.7 + stage: deploy + script: + - apt-get update -qq && apt-get install -y git # needed by Documenter + - julia -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("MyPackage");' # rebuild Julia (can be put somewhere else I'm sure + - julia -e 'using Pkg; import MyPackage; Pkg.add("Documenter")' # install Documenter + - julia --color=yes docs/make.jl # make documentation + - mv docs/build public # move to the directory picked up by Gitlab pages + artifacts: + paths: + - public + only: + - master + + +# WARNING: This template is using the `julia` images from [Docker +# Hub][3]. One can use custom Julia images and/or the official ones found +# in the same place. However, care must be taken to correctly locate the binary +# file (`/opt/julia/bin/julia` above), which is usually given on the image's +# description page. +# +# [3]: https://hub.docker.com/_/julia/ diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml new file mode 100644 index 00000000000..a4aed36889e --- /dev/null +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -0,0 +1,11 @@ +# use docker image with latex preinstalled +# since there is no official latex image, use https://github.com/blang/latex-docker +# possible alternative: https://github.com/natlownes/docker-latex +image: blang/latex + +build: + script: + - latexmk -pdf + artifacts: + paths: + - "*.pdf" diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml new file mode 100644 index 00000000000..d0cad285572 --- /dev/null +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -0,0 +1,85 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/php +image: php:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + +variables: + MYSQL_DATABASE: project_name + MYSQL_ROOT_PASSWORD: secret + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - vendor/ + - node_modules/ + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + # Update packages + - apt-get update -yqq + + # Prep for Node + - apt-get install gnupg -yqq + + # Upgrade to Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + + # Install dependencies + - apt-get install git nodejs libcurl4-gnutls-dev libicu-dev libmcrypt-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libpq-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev -yqq + + # Install php extensions + - docker-php-ext-install mbstring pdo_mysql curl json intl gd xml zip bz2 opcache + + # Install & enable Xdebug for code coverage reports + - pecl install xdebug + - docker-php-ext-enable xdebug + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + # comment this out if you don't have a node dependency + - npm install + + # Copy over testing configuration. + # Don't forget to set the database config in .env.testing correctly + # DB_HOST=mysql + # DB_DATABASE=project_name + # DB_USERNAME=root + # DB_PASSWORD=secret + - cp .env.testing .env + + # Run npm build + # comment this out if you don't have a frontend build + # you can change this to to your frontend building script like + # npm run build + - npm run dev + + # Generate an application key. Re-cache. + - php artisan key:generate + - php artisan config:cache + + # Run database migrations. + - php artisan migrate + + # Run database seed + - php artisan db:seed + +test: + script: + # run laravel tests + - php vendor/bin/phpunit --coverage-text --colors=never + + # run frontend tests + # if you have any task for testing frontend + # set it in your package.json script + # comment this out if you don't have a frontend test + - npm test diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml new file mode 100644 index 00000000000..492b3d03db2 --- /dev/null +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -0,0 +1,102 @@ +--- +# Build JAVA applications using Apache Maven (http://maven.apache.org) +# For docker image tags see https://hub.docker.com/_/maven/ +# +# For general lifecycle information see https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +# +# This template will build and test your projects as well as create the documentation. +# +# * Caches downloaded dependencies and plugins between invocation. +# * Verify but don't deploy merge requests. +# * Deploy built artifacts from master branch only. +# * Shows how to use multiple jobs in test stage for verifying functionality +# with multiple JDKs. +# * Uses site:stage to collect the documentation for multi-module projects. +# * Publishes the documentation for `master` branch. + +variables: + # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. + # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. + MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" + # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + # when running from the command line. + # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. + MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + +# Cache downloaded dependencies and plugins between builds. +# To keep cache across branches add 'key: "$CI_JOB_NAME"' +cache: + paths: + - .m2/repository + +# This will only validate and compile stuff and run e.g. maven-enforcer-plugin. +# Because some enforcer rules might check dependency convergence and class duplications +# we use `test-compile` here instead of `validate`, so the correct classpath is picked up. +.validate: &validate + stage: build + script: + - 'mvn $MAVEN_CLI_OPTS test-compile' + +# For merge requests do not `deploy` but only run `verify`. +# See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html +.verify: &verify + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS verify site site:stage' + except: + - master + +# Validate merge requests using JDK7 +validate:jdk7: + <<: *validate + image: maven:3.3.9-jdk-7 + +# Validate merge requests using JDK8 +validate:jdk8: + <<: *validate + image: maven:3.3.9-jdk-8 + +# Verify merge requests using JDK7 +verify:jdk7: + <<: *verify + image: maven:3.3.9-jdk-7 + +# Verify merge requests using JDK8 +verify:jdk8: + <<: *verify + image: maven:3.3.9-jdk-8 + + +# For `master` branch run `mvn deploy` automatically. +# Here you need to decide whether you want to use JDK7 or 8. +# To get this working you need to define a volume while configuring your gitlab-ci-multi-runner. +# Mount your `settings.xml` as `/root/.m2/settings.xml` which holds your secrets. +# See https://maven.apache.org/settings.html +deploy:jdk8: + # Use stage test here, so the pages job may later pickup the created site. + stage: test + script: + - 'mvn $MAVEN_CLI_OPTS deploy site site:stage' + only: + - master + # Archive up the built documentation site. + artifacts: + paths: + - target/staging + image: maven:3.3.9-jdk-8 + + +pages: + image: busybox:latest + stage: deploy + script: + # Because Maven appends the artifactId automatically to the staging path if you did define a parent pom, + # you might need to use `mv target/staging/YOUR_ARTIFACT_ID public` instead. + - mv target/staging public + dependencies: + - deploy:jdk8 + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml new file mode 100644 index 00000000000..3585f99760f --- /dev/null +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -0,0 +1,42 @@ +# This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) +# using the official mono docker image to build a visual studio project. +# +# MyProject.sln +# MyProject\ +# MyProject\ +# MyProject.csproj (console application) +# MyProject.Test\ +# MyProject.Test.csproj (test library using nuget packages "NUnit" and "NUnit.ConsoleRunner") +# +# Please find the full example project here: +# https://gitlab.com/tobiaskoch/gitlab-ci-example-mono + +# see https://hub.docker.com/_/mono/ +image: mono:latest + +stages: + - test + - deploy + +before_script: + - nuget restore -NonInteractive + +release: + stage: deploy + only: + - master + artifacts: + paths: + - build/release/MyProject.exe + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Release" /p:Platform="Any CPU" + /p:OutputPath="./../../build/release/" "MyProject.sln" + +debug: + stage: test + script: + # The output path is relative to the position of the csproj-file + - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" + /p:OutputPath="./../../build/debug/" "MyProject.sln" + - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml new file mode 100644 index 00000000000..41de1458582 --- /dev/null +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Official framework image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/node/tags/ +image: node:latest + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +test_async: + script: + - npm install + - node ./specs/start.js ./specs/async.spec.js + +test_db: + script: + - npm install + - node ./specs/start.js ./specs/db-postgres.spec.js diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml new file mode 100644 index 00000000000..290b9997084 --- /dev/null +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -0,0 +1,92 @@ +image: ayufan/openshift-cli + +stages: + - test + - review + - staging + - production + - cleanup + +variables: + OPENSHIFT_SERVER: openshift.default.svc.cluster.local + # OPENSHIFT_DOMAIN: apps.example.com + # Configure this variable in Secure Variables: + # OPENSHIFT_TOKEN: my.openshift.token + +test1: + stage: test + before_script: [] + script: + - echo run tests + +test2: + stage: test + before_script: [] + script: + - echo run tests + +.deploy: &deploy + before_script: + - oc login "$OPENSHIFT_SERVER" --token="$OPENSHIFT_TOKEN" --insecure-skip-tls-verify + - oc project "$CI_PROJECT_NAME-$CI_PROJECT_ID" 2> /dev/null || oc new-project "$CI_PROJECT_NAME-$CI_PROJECT_ID" + script: + - "oc get services $APP 2> /dev/null || oc new-app . --name=$APP --strategy=docker" + - "oc start-build $APP --from-dir=. --follow || sleep 3s && oc start-build $APP --from-dir=. --follow" + - "oc get routes $APP 2> /dev/null || oc expose service $APP --hostname=$APP_HOST" + +review: + <<: *deploy + stage: review + variables: + APP: review-$CI_COMMIT_REF_NAME + APP_HOST: $CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN + on_stop: stop-review + only: + - branches + except: + - master + +stop-review: + <<: *deploy + stage: cleanup + script: + - oc delete all -l "app=$APP" + when: manual + variables: + APP: review-$CI_COMMIT_REF_NAME + GIT_STRATEGY: none + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + only: + - branches + except: + - master + +staging: + <<: *deploy + stage: staging + variables: + APP: staging + APP_HOST: $CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + environment: + name: staging + url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN + only: + - master + +production: + <<: *deploy + stage: production + variables: + APP: production + APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + when: manual + environment: + name: production + url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN + only: + - master diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml new file mode 100644 index 00000000000..33f44ee9222 --- /dev/null +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Select image from https://hub.docker.com/_/php/ +image: php:7.1.1 + +# Select what we should cache between builds +cache: + paths: + - vendor/ + +before_script: +- apt-get update -yqq +- apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev +# Install PHP extensions +- docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache +# Install & enable Xdebug for code coverage reports +- pecl install xdebug +- docker-php-ext-enable xdebug +# Install and run Composer +- curl -sS https://getcomposer.org/installer | php +- php composer.phar install + +# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +services: + - mysql:5.7 + +# Set any variables we need +variables: + # Configure mysql environment variables (https://hub.docker.com/r/_/mysql/) + MYSQL_DATABASE: mysql_database + MYSQL_ROOT_PASSWORD: mysql_strong_password + +# Run our tests +# If Xdebug was installed you can generate a coverage report and see code coverage metrics. +test: + script: + - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml new file mode 100644 index 00000000000..fa296057c72 --- /dev/null +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: + name: hashicorp/packer:1.0.4 + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +before_script: + - packer --version + +stages: + - validate + - deploy + +validate: + stage: validate + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer validate + +build: + stage: deploy + environment: production + script: + - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml new file mode 100644 index 00000000000..7fcc0b436b5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/brunch +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g brunch + - brunch build --production + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml new file mode 100644 index 00000000000..791afdd23f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -0,0 +1,13 @@ +# Full project: https://gitlab.com/pages/doxygen +image: alpine + +pages: + script: + - apk update && apk add doxygen + - doxygen doxygen/Doxyfile + - mv doxygen/documentation/html/ public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml new file mode 100644 index 00000000000..9df2a4797b2 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -0,0 +1,17 @@ +image: node:latest + +# This folder is cached between builds +# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +cache: + paths: + - node_modules/ + +pages: + script: + - yarn install + - ./node_modules/.bin/gatsby build --prefix-paths + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml new file mode 100644 index 00000000000..249a168aa33 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/plain-html +pages: + stage: deploy + script: + - mkdir .public + - cp -r * .public + - mv .public public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml new file mode 100644 index 00000000000..dd3ef149668 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/harp +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules + + script: + - npm install -g harp + - harp compile ./ public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml new file mode 100644 index 00000000000..02d02250bbf --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -0,0 +1,16 @@ +# Full project: https://gitlab.com/pages/hexo +image: node:6.10.0 + +pages: + script: + - npm install + - ./node_modules/hexo/bin/hexo generate + artifacts: + paths: + - public + cache: + paths: + - node_modules + key: project + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml new file mode 100644 index 00000000000..b8cfb0f56f6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/hugo +image: dettmering/hugo-build + +pages: + script: + - hugo + artifacts: + paths: + - public + only: + - master + +test: + script: + - hugo + except: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml new file mode 100644 index 00000000000..f5b40f2b9f1 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -0,0 +1,25 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +cache: + paths: + - vendor/ + +test: + stage: test + script: + - pip install hyde + - hyde gen + except: + - master + +pages: + stage: deploy + script: + - pip install hyde + - hyde gen -d public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml new file mode 100644 index 00000000000..7abfaf53e8e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -0,0 +1,32 @@ +# This template uses the java:8 docker image because there isn't any +# official JBake image at this moment +# +# JBake https://jbake.org/ is a Java based, open source, static site/blog generator for developers & designers +# +# This yml works with jBake 2.5.1 +# Feel free to change JBAKE_VERSION version +# +# HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ + +image: java:8 + +variables: + JBAKE_VERSION: 2.5.1 + + +# We use SDKMan as tool for managing versions +before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION + +# This build job produced the output directory of your site +pages: + script: + - jbake . public + artifacts: + paths: + - public
\ No newline at end of file diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml new file mode 100644 index 00000000000..37f50554036 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Template project: https://gitlab.com/pages/jekyll +# Docs: https://docs.gitlab.com/ce/pages/ +image: ruby:2.3 + +variables: + JEKYLL_ENV: production + +before_script: +- bundle install + +test: + stage: test + script: + - bundle exec jekyll build -d test + artifacts: + paths: + - test + except: + - master + +pages: + stage: deploy + script: + - bundle exec jekyll build -d public + artifacts: + paths: + - public + only: + - master + diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml new file mode 100644 index 00000000000..0e5fb410a4e --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -0,0 +1,42 @@ +# Jigsaw is a simple static sites generator with Laravel's Blade. +# +# Full project: https://github.com/tightenco/jigsaw + +image: php:7.2 + +# These folders are cached between builds +cache: + paths: + - vendor/ + - node_modules/ + +before_script: + # Update packages + - apt-get update -yqq + + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + + # Install php extensions + - docker-php-ext-install zip + + # Install Composer and project dependencies. + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + + # Install Node dependencies. + - npm install + +pages: + script: + - npm run production + - mv build_production public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml new file mode 100644 index 00000000000..c5c44a5d86c --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/hyde +image: python:2.7 + +pages: + script: + - pip install lektor + - lektor build --output-path public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml new file mode 100644 index 00000000000..50e8b7ccd46 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Full project: https://gitlab.com/pages/metalsmith +image: node:4.2.2 + +pages: + cache: + paths: + - node_modules/ + + script: + - npm install -g metalsmith + - npm install + - make build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml new file mode 100644 index 00000000000..9f4cc0574d6 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Full project: https://gitlab.com/pages/middleman +image: ruby:2.3 + +cache: + paths: + - vendor + +test: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + except: + - master + +pages: + script: + - apt-get update -yqqq + - apt-get install -y nodejs + - bundle install --path vendor + - bundle exec middleman build + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml new file mode 100644 index 00000000000..b469b316ba5 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -0,0 +1,12 @@ +# Full project: https://gitlab.com/pages/nanoc +image: ruby:2.3 + +pages: + script: + - bundle install -j4 + - nanoc + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml new file mode 100644 index 00000000000..4762ec9acfd --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -0,0 +1,15 @@ +# Full project: https://gitlab.com/pages/octopress +image: ruby:2.3 + +pages: + script: + - apt-get update -qq && apt-get install -qq nodejs + - bundle install -j4 + - bundle exec rake generate + - mv public .public + - mv .public/octopress public + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml new file mode 100644 index 00000000000..c5f3154f587 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -0,0 +1,10 @@ +# Full project: https://gitlab.com/pages/pelican +image: python:2.7-alpine + +pages: + script: + - pip install -r requirements.txt + - pelican -s publishconf.py + artifacts: + paths: + - public/ diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml new file mode 100644 index 00000000000..098abe4daf5 --- /dev/null +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -0,0 +1,51 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/python/tags/ +image: python:latest + +# Change pip's cache directory to be inside the project directory since we can +# only cache local items. +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +# Pip's cache doesn't store the python packages +# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# +# If you want to also cache the installed packages, you have to install +# them in a virtualenv and cache it as well. +cache: + paths: + - .cache/pip + - venv/ + +before_script: + - python -V # Print out python version for debugging + - pip install virtualenv + - virtualenv venv + - source venv/bin/activate + +test: + script: + - python setup.py test + - pip install tox flake8 # you can also use tox + - tox -e py36,flake8 + +run: + script: + - python setup.py bdist_wheel + # an alternative approach is to install and run: + - pip install dist/* + # run the command here + artifacts: + paths: + - dist/*.whl + +pages: + script: + - pip install sphinx sphinx-rtd-theme + - cd doc ; make html + - mv build/html/ ../public/ + artifacts: + paths: + - public + only: + - master diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml new file mode 100644 index 00000000000..0d12cbc6460 --- /dev/null +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -0,0 +1,53 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/ruby/tags/ +image: "ruby:2.5" + +# Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +services: + - mysql:latest + - redis:latest + - postgres:latest + +variables: + POSTGRES_DB: database_name + +# Cache gems in between builds +cache: + paths: + - vendor/ruby + +# This is a basic example for a gem or script which doesn't use +# services such as redis or postgres +before_script: + - ruby -v # Print out ruby version for debugging + # Uncomment next line if your rails app needs a JS runtime: + # - apt-get update -q && apt-get install nodejs -yqq + - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + +# Optional - Delete if not using `rubocop` +rubocop: + script: + - rubocop + +rspec: + script: + - rspec spec + +rails: + variables: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB" + script: + - rails db:migrate + - rails db:seed + - rails test + +# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk +# are supported too: https://github.com/travis-ci/dpl +deploy: + type: deploy + environment: production + script: + - gem install dpl + - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml new file mode 100644 index 00000000000..cab087c48c7 --- /dev/null +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -0,0 +1,23 @@ +# Official language image. Look for the different tagged releases at: +# https://hub.docker.com/r/library/rust/tags/ +image: "rust:latest" + +# Optional: Pick zero or more services to be used on all builds. +# Only needed when using a docker container to run your tests in. +# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +#services: +# - mysql:latest +# - redis:latest +# - postgres:latest + +# Optional: Install a C compiler, cmake and git into the container. +# You will often need this when you (or any of your dependencies) depends on C code. +#before_script: +#- apt-get update -yqq +#- apt-get install -yqq --no-install-recommends build-essential + +# Use cargo to test the project +test:cargo: + script: + - rustc --version && cargo --version # Print version info for debugging + - cargo test --all --verbose diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml new file mode 100644 index 00000000000..b4208ed9d7d --- /dev/null +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -0,0 +1,22 @@ +# Official Java image. Look for the different tagged releases at +# https://hub.docker.com/r/library/java/tags/ . A Java image is not required +# but an image with a JVM speeds up the build a bit. +image: java:8 + +before_script: + # Enable the usage of sources over https + - apt-get update -yqq + - apt-get install apt-transport-https -yqq + # Add keyserver for SBT + - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + # Install SBT + - apt-get update -yqq + - apt-get install sbt -yqq + # Log the sbt version + - sbt sbt-version + +test: + script: + # Execute your project's tests + - sbt clean test diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml new file mode 100644 index 00000000000..ba8a802ba4f --- /dev/null +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -0,0 +1,30 @@ +# Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ +# This file assumes an own GitLab CI runner, setup on a macOS system. +stages: + - build + - archive + +build_project: + stage: build + script: + - xcodebuild clean -project ProjectName.xcodeproj -scheme SchemeName | xcpretty + - xcodebuild test -project ProjectName.xcodeproj -scheme SchemeName -destination 'platform=iOS Simulator,name=iPhone 8,OS=11.3' | xcpretty -s + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 + +archive_project: + stage: archive + script: + - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName + - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" + only: + - master + artifacts: + paths: + - build/ProjectName.ipa + tags: + - ios_11-3 + - xcode_9-3 + - macos_10-13 diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml new file mode 100644 index 00000000000..7160fce26a8 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -0,0 +1,55 @@ +# Official image for Hashicorp's Terraform. It uses light image which is Alpine +# based as it is much lighter. +# +# Entrypoint is also needed as image by default set `terraform` binary as an +# entrypoint. +image: + name: hashicorp/terraform:light + entrypoint: + - '/usr/bin/env' + - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +# Default output file for Terraform plan +variables: + PLAN: plan.tfplan + +cache: + paths: + - .terraform + +before_script: + - terraform --version + - terraform init + +stages: + - validate + - build + - deploy + +validate: + stage: validate + script: + - terraform validate + +plan: + stage: build + script: + - terraform plan -out=$PLAN + artifacts: + name: plan + paths: + - $PLAN + +# Separate apply job for manual launching Terraform as it can be destructive +# action. +apply: + stage: deploy + environment: + name: production + script: + - terraform apply -input=false $PLAN + dependencies: + - plan + when: manual + only: + - master diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml new file mode 100644 index 00000000000..fc3d4ecdbba --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -0,0 +1,86 @@ +# The following script will work for any project that can be built from command line by msbuild +# It uses powershell shell executor, so you need to add the following line to your config.toml file +# (located in gitlab-runner.exe directory): +# shell = "powershell" +# +# The script is composed of 3 stages: build, test and deploy. +# +# The build stage restores NuGet packages and uses msbuild to build the exe and msi +# One major issue you'll find is that you can't build msi projects from command line +# if you use vdproj. There are workarounds building msi via devenv, but they rarely work +# The best solution is migrating your vdproj projects to WiX, as it can be build directly +# by msbuild. +# +# The test stage runs nunit from command line against Test project inside your solution +# It also saves the resulting TestResult.xml file +# +# The deploy stage copies the exe and msi from build stage to a network drive +# You need to have the network drive mapped as Local System user for gitlab-runner service to see it +# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes + + +# place project specific paths in variables to make the rest of the script more generic +variables: + EXE_RELEASE_FOLDER: 'YourApp\bin\Release' + MSI_RELEASE_FOLDER: 'Setup\bin\Release' + TEST_FOLDER: 'Tests\bin\Release' + DEPLOY_FOLDER: 'P:\Projects\YourApp\Builds' + + NUGET_PATH: 'C:\NuGet\nuget.exe' + MSBUILD_PATH: 'C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe' + NUNIT_PATH: 'C:\Program Files (x86)\NUnit.org\nunit-console\nunit3-console.exe' + +stages: + - build + - test + - deploy + +build_job: + stage: build + only: + - tags # the build process will only be started by git tag commits + script: + - '& "$env:NUGET_PATH" restore' # restore Nuget dependencies + - '& "$env:MSBUILD_PATH" /p:Configuration=Release' # build the project + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '$env:EXE_RELEASE_FOLDER\YourApp.exe' # saving exe to copy to deploy folder + - '$env:MSI_RELEASE_FOLDER\YourApp Setup.msi' # saving msi to copy to deploy folder + - '$env:TEST_FOLDER\' # saving entire Test project so NUnit can run tests + +test_job: + stage: test + only: + - tags + script: + - '& "$env:NUNIT_PATH" ".\$env:TEST_FOLDER\Tests.dll"' # running NUnit tests + artifacts: + expire_in: 1 week # save gitlab server space, we copy the files we need to deploy folder later on + paths: + - '.\TestResult.xml' # saving NUnit results to copy to deploy folder + dependencies: + - build_job + +deploy_job: + stage: deploy + only: + - tags + script: + # Compose a folder for each release based on commit tag. + # Assuming your tag is Rev1.0.0.1, and your last commit message is 'First commit' + # the artifact files will be copied to: + # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ + - '$commitSubject = git log -1 --pretty=%s' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + + # xcopy takes care of recursively creating required folders + - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' + - 'xcopy /y ".\$env:MSI_RELEASE_FOLDER\YourApp Setup.msi" "$deployFolder"' + - 'xcopy /y ".\TestResult.xml" "$deployFolder"' + + dependencies: + - build_job + - test_job +
\ No newline at end of file diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index fe15fabc2e8..bf5f2a31f0e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,7 +1,16 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace + include ::Gitlab::ExclusiveLeaseHelpers + + LOCK_TTL = 1.minute + LOCK_RETRIES = 2 + LOCK_SLEEP = 0.001.seconds + ArchiveError = Class.new(StandardError) + AlreadyArchivedError = Class.new(StandardError) attr_reader :job @@ -75,9 +84,39 @@ module Gitlab stream&.close end - def write(mode) + def write(mode, &blk) + in_write_lock do + unsafe_write!(mode, &blk) + end + end + + def erase! + ## + # Erase the archived trace + trace_artifact&.destroy! + + ## + # Erase the live trace + job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace + job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace + ensure + @current_path = nil + end + + def archive! + in_write_lock do + unsafe_archive! + end + end + + private + + def unsafe_write!(mode, &blk) stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + raise AlreadyArchivedError, 'Could not write to the archived trace' + elsif current_path File.open(current_path, mode) elsif Feature.enabled?('ci_enable_live_trace') Gitlab::Ci::Trace::ChunkedIO.new(job) @@ -93,19 +132,8 @@ module Gitlab stream&.close end - def erase! - trace_artifact&.destroy - - paths.each do |trace_path| - FileUtils.rm(trace_path, force: true) - end - - job.trace_chunks.fast_destroy_all - job.erase_old_trace! - end - - def archive! - raise ArchiveError, 'Already archived' if trace_artifact + def unsafe_archive! + raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? if job.trace_chunks.any? @@ -126,7 +154,10 @@ module Gitlab end end - private + def in_write_lock(&blk) + lock_key = "trace:write:lock:#{job.id}" + in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk) + end def archive_stream!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| @@ -148,6 +179,8 @@ module Gitlab def create_build_trace!(job, path) File.open(path) do |stream| + # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration + # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/20307 job.create_job_artifacts_trace!( project: job.project, file_type: :trace, diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index bfe0c2a2c26..8c6fd56493f 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ## # This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) # source: https://gitlab.com/snippets/1685610 @@ -11,7 +13,7 @@ module Gitlab attr_reader :build attr_reader :tell, :size - attr_reader :chunk, :chunk_range + attr_reader :chunk_data, :chunk_range alias_method :pos, :tell @@ -66,42 +68,46 @@ module Gitlab end end - def read(length = nil, outbuf = "") - out = "" + def read(length = nil, outbuf = nil) + out = [] length ||= size - tell until length <= 0 || eof? data = chunk_slice_from_offset - break if data.empty? + raise FailedToGetChunkError if data.empty? chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min - chunk_data = data.byteslice(0, chunk_bytes) + chunk_data_slice = data.byteslice(0, chunk_bytes) - out << chunk_data - @tell += chunk_data.bytesize - length -= chunk_data.bytesize + out << chunk_data_slice + @tell += chunk_data_slice.bytesize + length -= chunk_data_slice.bytesize end + out = out.join + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out + outbuf.replace(out) end out end def readline - out = "" + out = [] until eof? data = chunk_slice_from_offset + raise FailedToGetChunkError if data.empty? + new_line = data.index("\n") if !new_line.nil? - out << data[0..new_line] - @tell += new_line + 1 + raw_data = data[0..new_line] + out << raw_data + @tell += raw_data.bytesize break else out << data @@ -109,7 +115,7 @@ module Gitlab end end - out + out.join end def write(data) @@ -118,13 +124,13 @@ module Gitlab while tell < start_pos + data.bytesize # get slice from current offset till the end where it falls into chunk chunk_bytes = CHUNK_SIZE - chunk_offset - chunk_data = data.byteslice(tell - start_pos, chunk_bytes) + data_slice = data.byteslice(tell - start_pos, chunk_bytes) # append data to chunk, overwriting from that point - ensure_chunk.append(chunk_data, chunk_offset) + ensure_chunk.append(data_slice, chunk_offset) # move offsets within buffer - @tell += chunk_data.bytesize + @tell += data_slice.bytesize @size = [size, tell].max end @@ -133,6 +139,7 @@ module Gitlab invalidate_chunk_cache end + # rubocop: disable CodeReuse/ActiveRecord def truncate(offset) raise ArgumentError, 'Outside of file' if offset > size || offset < 0 return if offset == size # Skip the following process as it doesn't affect anything @@ -148,6 +155,7 @@ module Gitlab ensure invalidate_chunk_cache end + # rubocop: enable CodeReuse/ActiveRecord def flush # no-op @@ -178,12 +186,12 @@ module Gitlab current_chunk.tap do |chunk| raise FailedToGetChunkError unless chunk - @chunk = chunk.data + @chunk_data = chunk.data @chunk_range = chunk.range end end - @chunk[chunk_offset..CHUNK_SIZE] + @chunk_data.byteslice(chunk_offset, CHUNK_SIZE) end def chunk_offset @@ -206,9 +214,11 @@ module Gitlab @chunks_cache = [] end + # rubocop: disable CodeReuse/ActiveRecord def current_chunk @chunks_cache[chunk_index] ||= trace_chunks.find_by(chunk_index: chunk_index) end + # rubocop: enable CodeReuse/ActiveRecord def build_chunk @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) @@ -218,13 +228,17 @@ module Gitlab current_chunk || build_chunk end + # rubocop: disable CodeReuse/ActiveRecord def trace_chunks ::Ci::BuildTraceChunk.where(build: build) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def calculate_size trace_chunks.order(chunk_index: :desc).first.try(&:end_offset).to_i end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb deleted file mode 100644 index 8788af57a67..00000000000 --- a/lib/gitlab/ci/trace/http_io.rb +++ /dev/null @@ -1,197 +0,0 @@ -## -# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) -# source: https://gitlab.com/snippets/1685610 -module Gitlab - module Ci - class Trace - class HttpIO - BUFFER_SIZE = 128.kilobytes - - InvalidURLError = Class.new(StandardError) - FailedToGetChunkError = Class.new(StandardError) - - attr_reader :uri, :size - attr_reader :tell - attr_reader :chunk, :chunk_range - - alias_method :pos, :tell - - def initialize(url, size) - raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) - - @uri = URI(url) - @size = size - @tell = 0 - end - - def close - # no-op - end - - def binmode - # no-op - end - - def binmode? - true - end - - def path - nil - end - - def url - @uri.to_s - end - - def seek(pos, where = IO::SEEK_SET) - new_pos = - case where - when IO::SEEK_END - size + pos - when IO::SEEK_SET - pos - when IO::SEEK_CUR - tell + pos - else - -1 - end - - raise 'new position is outside of file' if new_pos < 0 || new_pos > size - - @tell = new_pos - end - - def eof? - tell == size - end - - def each_line - until eof? - line = readline - break if line.nil? - - yield(line) - end - end - - def read(length = nil, outbuf = "") - out = "" - - length ||= size - tell - - until length <= 0 || eof? - data = get_chunk - break if data.empty? - - chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min - chunk_data = data.byteslice(0, chunk_bytes) - - out << chunk_data - @tell += chunk_data.bytesize - length -= chunk_data.bytesize - end - - # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality - if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out - end - - out - end - - def readline - out = "" - - until eof? - data = get_chunk - new_line = data.index("\n") - - if !new_line.nil? - out << data[0..new_line] - @tell += new_line + 1 - break - else - out << data - @tell += data.bytesize - end - end - - out - end - - def write(data) - raise NotImplementedError - end - - def truncate(offset) - raise NotImplementedError - end - - def flush - raise NotImplementedError - end - - def present? - true - end - - private - - ## - # The below methods are not implemented in IO class - # - def in_range? - @chunk_range&.include?(tell) - end - - def get_chunk - unless in_range? - response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| - http.request(request) - end - - raise FailedToGetChunkError unless response.code == '200' || response.code == '206' - - @chunk = response.body.force_encoding(Encoding::BINARY) - @chunk_range = response.content_range - - ## - # Note: If provider does not return content_range, then we set it as we requested - # Provider: minio - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: AWS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # Provider: GCS - # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206 - # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200 - @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) - end - - @chunk[chunk_offset..BUFFER_SIZE] - end - - def request - Net::HTTP::Get.new(uri).tap do |request| - request.set_range(chunk_start, BUFFER_SIZE) - end - end - - def chunk_offset - tell % BUFFER_SIZE - end - - def chunk_start - (tell / BUFFER_SIZE) * BUFFER_SIZE - end - - def chunk_end - [chunk_start + BUFFER_SIZE, size].min - end - end - end - end -end diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index 9bb0166c9e3..f33f8cc56c1 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace @@ -75,19 +77,19 @@ module Gitlab @beginning_of_section_regex ||= /section_/.freeze end - def find_next_marker(s) + def find_next_marker(scanner) beginning_of_section_len = 8 - maybe_marker = s.exist?(beginning_of_section_regex) + maybe_marker = scanner.exist?(beginning_of_section_regex) if maybe_marker.nil? - s.terminate + scanner.terminate else # repositioning at the beginning of the match - s.pos += maybe_marker - beginning_of_section_len + scanner.pos += maybe_marker - beginning_of_section_len if block_given? - good_marker = yield(s) + good_marker = yield(scanner) # if not a good marker: Consuming the matched beginning_of_section_regex - s.pos += beginning_of_section_len unless good_marker + scanner.pos += beginning_of_section_len unless good_marker end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index a71040e5e56..e61fb50a303 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace @@ -41,19 +43,14 @@ module Gitlab def append(data, offset) data = data.force_encoding(Encoding::BINARY) - stream.truncate(offset) - stream.seek(0, IO::SEEK_END) + stream.seek(offset, IO::SEEK_SET) stream.write(data) - stream.flush() + stream.truncate(offset + data.bytesize) + stream.flush end def set(data) - data = data.force_encoding(Encoding::BINARY) - - stream.seek(0, IO::SEEK_SET) - stream.write(data) - stream.truncate(data.bytesize) - stream.flush() + append(data, 0) end def raw(last_lines: nil) @@ -129,8 +126,7 @@ module Gitlab debris = '' until (buf = read_backward(BUFFER_SIZE)).empty? - buf += debris - debris, *lines = buf.each_line.to_a + debris, *lines = (buf + debris).each_line.to_a lines.reverse_each do |line| yield(line.force_encoding(Encoding.default_external)) end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index ad30b3f427c..a7b4e0348c2 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index d00e5b07f95..e3e4e62cc02 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -1,9 +1,14 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables class Collection class Item def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { key: key, value: value, public: public, file: file } @@ -31,7 +36,7 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource) + self.new(resource.symbolize_keys) when ::HasVariable self.new(resource.to_runner_variable) when self diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index e829f2a95f8..07ba6f83d47 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -1,14 +1,16 @@ +# frozen_string_literal: true + module Gitlab module Ci class YamlProcessor ValidationError = Class.new(StandardError) - include Gitlab::Ci::Config::Entry::LegacyValidationHelpers + include Gitlab::Config::Entry::LegacyValidationHelpers attr_reader :cache, :stages, :jobs def initialize(config, opts = {}) - @ci_config = Gitlab::Ci::Config.new(config, opts) + @ci_config = Gitlab::Ci::Config.new(config, **opts) @config = @ci_config.to_hash unless @ci_config.valid? @@ -16,7 +18,7 @@ module Gitlab end initial_parsing - rescue Gitlab::Ci::Config::Loader::FormatError => e + rescue Gitlab::Ci::Config::ConfigError => e raise ValidationError, e.message end @@ -31,8 +33,7 @@ module Gitlab { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - commands: job[:commands], - tag_list: job[:tags] || [], + tag_list: job[:tags], name: job[:name].to_s, allow_failure: job[:ignore], when: job[:when] || 'on_success', @@ -49,8 +50,12 @@ module Gitlab script: job[:script], after_script: job[:after_script], environment: job[:environment], - retry: job[:retry] - }.compact } + retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], + start_in: job[:start_in], + trigger: job[:trigger] + }.compact }.compact end def stage_builds_attributes(stage) @@ -101,7 +106,7 @@ module Gitlab ## # Jobs # - @jobs = @ci_config.jobs + @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs @jobs.each do |name, job| # logical validation for job diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb index def1373d8cf..d5d3eb804ae 100644 --- a/lib/gitlab/ci_access.rb +++ b/lib/gitlab/ci_access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # For backwards compatibility, generic CI (which is a build without a user) is # allowed to :build_download_code without any other checks. diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb new file mode 100644 index 00000000000..2ee8b60e76a --- /dev/null +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploadFileFinder + FIND_BATCH_SIZE = 500 + ABSOLUTE_UPLOAD_DIR = FileUploader.root.freeze + EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*".freeze + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + + # Paths are relative to the upload directory + def each_file_batch(batch_size: FIND_BATCH_SIZE, &block) + cmd = build_find_command(ABSOLUTE_UPLOAD_DIR) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + private + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) if paths.any? + end + + def build_find_command(search_dir) + cmd = %W[find -L #{search_dir} + -type f + ! ( -path #{EXCLUDED_SYSTEM_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] + + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice + + log_msg = "find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg + + cmd + end + + def which_ionice + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, + # even though it isn't very nice + false + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb new file mode 100644 index 00000000000..82a405362c2 --- /dev/null +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploads + LOST_AND_FOUND = File.join(ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR, '-', 'project-lost-found') + + attr_reader :logger + + def initialize(logger: nil) + @logger = logger || Rails.logger + end + + def run!(dry_run: true) + logger.info "Looking for orphaned project uploads to clean up#{'. Dry run' if dry_run}..." + + each_orphan_file do |path, upload_path| + result = cleanup(path, upload_path, dry_run) + + logger.info result + end + end + + private + + def cleanup(path, upload_path, dry_run) + # This happened in staging: + # `find` returned a path on which `File.delete` raised `Errno::ENOENT` + return "Cannot find file: #{path}" unless File.exist?(path) + + correct_path = upload_path && find_correct_path(upload_path) + + if correct_path + move(path, correct_path, 'fix', dry_run) + else + move_to_lost_and_found(path, dry_run) + end + end + + # Accepts a path in the form of "#{hex_secret}/#{filename}" + # rubocop: disable CodeReuse/ActiveRecord + def find_correct_path(upload_path) + upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) + return unless upload && upload.local? && upload.model + + upload.absolute_path + rescue => e + logger.error e.message + + # absolute_path depends on a lot of code. If it doesn't work, then it + # it doesn't matter if the upload file is in the right place. Treat it + # as uncorrectable. + # I.e. the project record might be missing, which raises an exception. + nil + end + # rubocop: enable CodeReuse/ActiveRecord + + def move_to_lost_and_found(path, dry_run) + new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND) + + move(path, new_path, 'move to lost and found', dry_run) + end + + def move(path, new_path, prefix, dry_run) + action = "#{prefix} #{path} -> #{new_path}" + + if dry_run + "Can #{action}" + else + begin + FileUtils.mkdir_p(File.dirname(new_path)) + FileUtils.mv(path, new_path) + + "Did #{action}" + rescue => e + "Error during #{action}: #{e.inspect}" + end + end + end + + # Yields absolute paths of project upload files that are not in the + # uploads table + def each_orphan_file + ProjectUploadFileFinder.new.each_file_batch do |file_paths| + logger.debug "Processing batch of #{file_paths.size} project upload file paths, starting with #{file_paths.first}" + + file_paths.each do |path| + pup = ProjectUploadPath.from_path(path) + + yield(path, pup.upload_path) if pup.orphan? + end + end + end + + class ProjectUploadPath + PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze + + attr_reader :full_path, :upload_path + + def initialize(full_path, upload_path) + @full_path = full_path + @upload_path = upload_path + end + + def self.from_path(path) + path_matched = path.match(PROJECT_FULL_PATH_REGEX) + return new(nil, nil) unless path_matched + + new(path_matched[1], path_matched[2]) + end + + # rubocop: disable CodeReuse/ActiveRecord + def orphan? + return true if full_path.nil? || upload_path.nil? + + # It's possible to reduce to one query, but `where_full_path_in` is complex + !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader') + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + # rubocop: disable CodeReuse/ActiveRecord + def project_id + @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb new file mode 100644 index 00000000000..03298d960a4 --- /dev/null +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +module Gitlab + module Cleanup + class RemoteUploads + attr_reader :logger + + BATCH_SIZE = 100 + + def initialize(logger: nil) + @logger = logger || Rails.logger + end + + def run!(dry_run: false) + unless configuration.enabled + logger.warn "Object storage not enabled. Exit".color(:yellow) + + return + end + + logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..." + + each_orphan_file do |file| + info = if dry_run + "Can be moved to lost and found: #{file.key}" + else + new_path = move_to_lost_and_found(file) + "Moved to lost and found: #{file.key} -> #{new_path}" + end + + logger.info(info) + end + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def each_orphan_file + # we want to skip files already moved to lost_and_found directory + lost_dir_match = "^#{lost_and_found_dir}\/" + + remote_directory.files.each_slice(BATCH_SIZE) do |remote_files| + remote_files.reject! { |file| file.key.match(/#{lost_dir_match}/) } + file_paths = remote_files.map(&:key) + tracked_paths = Upload + .where(store: ObjectStorage::Store::REMOTE, path: file_paths) + .pluck(:path) + + remote_files.reject! { |file| tracked_paths.include?(file.key) } + remote_files.each do |file| + yield file + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def move_to_lost_and_found(file) + new_path = "#{lost_and_found_dir}/#{file.key}" + + file.copy(configuration['remote_directory'], new_path) + file.destroy + + new_path + end + + def lost_and_found_dir + 'lost_and_found' + end + + def remote_directory + connection.directories.new(key: configuration['remote_directory']) + end + + def connection + ::Fog::Storage.new(configuration['connection'].symbolize_keys) + end + + def configuration + Gitlab.config.uploads.object_store + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 7e7aaeeaa17..4ba921569ad 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ClosingIssueExtractor ISSUE_CLOSING_REGEX = begin diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb new file mode 100644 index 00000000000..b05dca409d1 --- /dev/null +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + # + # LifecycleEvents lets Rails initializers register application startup hooks + # that are sensitive to forking. For example, to defer the creation of + # watchdog threads. This lets us abstract away the Unix process + # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # + # We have three lifecycle events. + # + # - before_fork (only in forking processes) + # - worker_start + # - before_master_restart (only in forking processes) + # + # Blocks will be executed in the order in which they are registered. + # + class LifecycleEvents + class << self + # + # Hook registration methods (called from initializers) + # + def on_worker_start(&block) + if in_clustered_environment? + # Defer block execution + (@worker_start_hooks ||= []) << block + else + yield + end + end + + def on_before_fork(&block) + return unless in_clustered_environment? + + # Defer block execution + (@before_fork_hooks ||= []) << block + end + + def on_master_restart(&block) + return unless in_clustered_environment? + + # Defer block execution + (@master_restart_hooks ||= []) << block + end + + # + # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # + def do_worker_start + @worker_start_hooks&.each do |block| + block.call + end + end + + def do_before_fork + @before_fork_hooks&.each do |block| + block.call + end + end + + def do_master_restart + @master_restart_hooks && @master_restart_hooks.each do |block| + block.call + end + end + + # Puma doesn't use singletons (which is good) but + # this means we need to pass through whether the + # puma server is running in single mode or cluster mode + def set_puma_options(options) + @puma_options = options + end + + private + + def in_clustered_environment? + # Sidekiq doesn't fork + return false if Sidekiq.server? + + # Unicorn always forks + return true if defined?(::Unicorn) + + # Puma sometimes forks + return true if in_clustered_puma? + + # Default assumption is that we don't fork + false + end + + def in_clustered_puma? + return false unless defined?(::Puma) + + @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb new file mode 100644 index 00000000000..4ed9a9a02ab --- /dev/null +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + class PumaWorkerKillerInitializer + def self.start(puma_options, puma_per_worker_max_memory_mb: 650) + require 'puma_worker_killer' + + PumaWorkerKiller.config do |config| + # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) + # Importantly RAM is for _all_workers (ie, the cluster), + # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX + worker_count = puma_options[:workers] || 1 + # The Puma Worker Killer checks the total RAM used by both the master + # and worker processes. Bump the limits to N+1 instead of N workers + # to account for this: + # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 + config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb + + config.frequency = 20 # seconds + + # We just want to limit to a fixed maximum, unrelated to the total amount + # of available RAM. + config.percent_usage = 0.98 + + # Ideally we'll never hit the maximum amount of memory. If so the worker + # is restarted already, thus periodically restarting workers shouldn't be + # needed. + config.rolling_restart_frequency = false + end + + PumaWorkerKiller.start + end + end + end +end diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index 9c4664df903..881e5dbc923 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Module containing GitLab's syntax color scheme definitions and helper # methods for accessing them. @@ -10,7 +12,8 @@ module Gitlab Scheme.new(2, 'Dark', 'dark'), Scheme.new(3, 'Solarized Light', 'solarized-light'), Scheme.new(4, 'Solarized Dark', 'solarized-dark'), - Scheme.new(5, 'Monokai', 'monokai') + Scheme.new(5, 'Monokai', 'monokai'), + Scheme.new(6, 'None', 'none') ].freeze # Convenience method to get a space-separated String of all the color scheme diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb new file mode 100644 index 00000000000..560fe63df0e --- /dev/null +++ b/lib/gitlab/config/entry/attributable.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Attributable + extend ActiveSupport::Concern + + class_methods do + def attributes(*attributes) + attributes.flatten.each do |attribute| + if method_defined?(attribute) + raise ArgumentError, 'Method already defined!' + end + + define_method(attribute) do + return unless config.is_a?(Hash) + + config[attribute] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/boolean.rb b/lib/gitlab/config/entry/boolean.rb new file mode 100644 index 00000000000..1e8a57356e3 --- /dev/null +++ b/lib/gitlab/config/entry/boolean.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents a boolean value. + # + class Boolean < Node + include Validatable + + validations do + validates :config, boolean: true + end + end + end + end +end diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb new file mode 100644 index 00000000000..37ba16dba25 --- /dev/null +++ b/lib/gitlab/config/entry/configurable.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + included do + include Validatable + + validations do + validates :config, type: Hash + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def compose!(deps = nil) + return unless valid? + + self.class.nodes.each do |key, factory| + factory + .value(config[key]) + .with(key: key, parent: self) + + entries[key] = factory.create! + end + + yield if block_given? + + entries.each_value do |entry| + entry.compose!(deps) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + class_methods do + def nodes + Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def entry(key, entry, metadata) + factory = ::Gitlab::Config::Entry::Factory.new(entry) + .with(description: metadata[:description]) + .with(default: metadata[:default]) + + (@nodes ||= {}).merge!(key.to_sym => factory) + end + # rubocop: enable CodeReuse/ActiveRecord + + def helpers(*nodes) + nodes.each do |symbol| + define_method("#{symbol}_defined?") do + entries[symbol]&.specified? + end + + define_method("#{symbol}_value") do + return unless entries[symbol] && entries[symbol].valid? + + entries[symbol].value + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb new file mode 100644 index 00000000000..79f9ff32514 --- /dev/null +++ b/lib/gitlab/config/entry/factory.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Factory class responsible for fabricating entry objects. + # + class Factory + InvalidFactory = Class.new(StandardError) + + def initialize(entry) + @entry = entry + @metadata = {} + @attributes = { default: entry.default } + end + + def value(value) + @value = value + self + end + + def metadata(metadata) + @metadata.merge!(metadata.compact) + self + end + + def with(attributes) + @attributes.merge!(attributes.compact) + self + end + + def create! + raise InvalidFactory unless defined?(@value) + + ## + # We assume that unspecified entry is undefined. + # See issue #18775. + # + if @value.nil? + Entry::Unspecified.new(fabricate_unspecified) + else + fabricate(@entry, @value) + end + end + + private + + def fabricate_unspecified + ## + # If entry has a default value we fabricate concrete node + # with default value. + # + default = @attributes.fetch(:default) + + if default.nil? + fabricate(Entry::Undefined) + else + fabricate(@entry, default) + end + end + + def fabricate(entry, value = nil) + entry.new(value, @metadata).tap do |node| + node.key = @attributes[:key] + node.parent = @attributes[:parent] + node.default = @attributes[:default] + node.description = @attributes[:description] + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb new file mode 100644 index 00000000000..d3ab5625743 --- /dev/null +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module LegacyValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_duration_limit(value, limit) + return false unless value.is_a?(String) + + ChronicDuration.parse(value).second.from_now < + ChronicDuration.parse(limit).second.from_now + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + validate_regexp(value[1...-1]) + else + true + end + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end +end diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb new file mode 100644 index 00000000000..9999ab4ff95 --- /dev/null +++ b/lib/gitlab/config/entry/node.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Base abstract class for each configuration entry node. + # + class Node + InvalidError = Class.new(StandardError) + + attr_reader :config, :metadata + attr_accessor :key, :parent, :default, :description + + def initialize(config, **metadata) + @config = config + @metadata = metadata + @entries = {} + + self.class.aspects.to_a.each do |aspect| + instance_exec(&aspect) + end + end + + def [](key) + @entries[key] || Entry::Undefined.new + end + + def compose!(deps = nil) + return unless valid? + + yield if block_given? + end + + def leaf? + @entries.none? + end + + def descendants + @entries.values + end + + def ancestors + @parent ? @parent.ancestors + [@parent] : [] + end + + def valid? + errors.none? + end + + def errors + [] + end + + def value + if leaf? + @config + else + meaningful = @entries.select do |_key, value| + value.specified? && value.relevant? + end + + Hash[meaningful.map { |key, entry| [key, entry.value] }] + end + end + + def specified? + true + end + + def relevant? + true + end + + def location + name = @key.presence || self.class.name.to_s.demodulize + .underscore.humanize.downcase + + ancestors.map(&:key).append(name).compact.join(':') + end + + def inspect + val = leaf? ? config : descendants + unspecified = specified? ? '' : '(unspecified) ' + "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" + end + + def self.default(**) + end + + def self.aspects + @aspects ||= [] + end + + private + + attr_reader :entries + end + end + end +end diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb new file mode 100644 index 00000000000..5fbf7565e2a --- /dev/null +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + class Simplifiable < SimpleDelegator + EntryStrategy = Struct.new(:name, :condition) + + attr_reader :subject + + def initialize(config, **metadata) + unless self.class.const_defined?(:UnknownStrategy) + raise ArgumentError, 'UndefinedStrategy not available!' + end + + strategy = self.class.strategies.find do |variant| + variant.condition.call(config) + end + + entry = self.class.entry_class(strategy) + + super(@subject = entry.new(config, metadata)) + end + + def self.strategy(name, **opts) + EntryStrategy.new(name, opts.fetch(:if)).tap do |strategy| + strategies.append(strategy) + end + end + + def self.strategies + @strategies ||= [] + end + + def self.entry_class(strategy) + if strategy.present? + self.const_get(strategy.name) + else + self::UnknownStrategy + end + end + + def self.default + end + end + end + end +end diff --git a/lib/gitlab/config/entry/undefined.rb b/lib/gitlab/config/entry/undefined.rb new file mode 100644 index 00000000000..5f708abc80c --- /dev/null +++ b/lib/gitlab/config/entry/undefined.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This class represents an undefined entry. + # + class Undefined < Node + def initialize(*) + super(nil) + end + + def value + nil + end + + def valid? + true + end + + def errors + [] + end + + def specified? + false + end + + def relevant? + false + end + + def inspect + "#<#{self.class.name}>" + end + end + end + end +end diff --git a/lib/gitlab/config/entry/unspecified.rb b/lib/gitlab/config/entry/unspecified.rb new file mode 100644 index 00000000000..c096180d0f8 --- /dev/null +++ b/lib/gitlab/config/entry/unspecified.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # This class represents an unspecified entry. + # + # It decorates original entry adding method that indicates it is + # unspecified. + # + class Unspecified < SimpleDelegator + def specified? + false + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validatable.rb b/lib/gitlab/config/entry/validatable.rb new file mode 100644 index 00000000000..1c88c68c11c --- /dev/null +++ b/lib/gitlab/config/entry/validatable.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validatable + extend ActiveSupport::Concern + + def self.included(node) + node.aspects.append -> do + @validator = self.class.validator.new(self) + @validator.validate(:new) + end + end + + def errors + @validator.messages + descendants.flat_map(&:errors) # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + class_methods do + def validator + @validator ||= Class.new(Entry::Validator).tap do |validator| + if defined?(@validations) + @validations.each { |rules| validator.class_eval(&rules) } + end + end + end + + private + + def validations(&block) + (@validations ||= []).append(block) + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validator.rb b/lib/gitlab/config/entry/validator.rb new file mode 100644 index 00000000000..e5efd4a7b0a --- /dev/null +++ b/lib/gitlab/config/entry/validator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + class Validator < SimpleDelegator + include ActiveModel::Validations + include Entry::Validators + + def initialize(entry) + super(entry) + end + + def messages + errors.full_messages.map do |error| + "#{location} #{error}".downcase + end + end + + def self.name + 'Validator' + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb new file mode 100644 index 00000000000..25bfa50f829 --- /dev/null +++ b/lib/gitlab/config/entry/validators.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validators + class AllowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unknown_keys = value.try(:keys).to_a - options[:in] + + if unknown_keys.any? + record.errors.add(attribute, "contains unknown keys: " + + unknown_keys.join(', ')) + end + end + end + + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + + class AllowedArrayValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unkown_values = value - options[:in] + unless unkown_values.empty? + record.errors.add(attribute, "contains unknown values: " + + unkown_values.join(', ')) + end + end + end + + class ArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_array_of_strings(value) + record.errors.add(attribute, 'should be an array of strings') + end + end + end + + class BooleanValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_boolean(value) + record.errors.add(attribute, 'should be a boolean value') + end + end + end + + class DurationValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_duration(value) + record.errors.add(attribute, 'should be a duration') + end + + if options[:limit] + unless validate_duration_limit(value, options[:limit]) + record.errors.add(attribute, 'should not exceed the limit') + end + end + end + end + + class HashOrStringValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(String) + record.errors.add(attribute, 'should be a hash or a string') + end + end + end + + class HashOrIntegerValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(Integer) + record.errors.add(attribute, 'should be a hash or an integer') + end + end + end + + class KeyValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + if validate_string(value) + validate_path(record, attribute, value) + else + record.errors.add(attribute, 'should be a string or symbol') + end + end + + private + + def validate_path(record, attribute, value) + path = CGI.unescape(value.to_s) + + if path.include?('/') + record.errors.add(attribute, 'cannot contain the "/" character') + elsif path == '.' || path == '..' + record.errors.add(attribute, 'cannot be "." or ".."') + end + end + end + + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + + true + end + end + + class ArrayOfStringsOrStringValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_string(value) + record.errors.add(attribute, 'should be an array of strings or a string') + end + end + + private + + def validate_array_of_strings_or_string(values) + validate_array_of_strings(values) || validate_string(values) + end + end + + class TypeValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + type = options[:with] + raise unless type.is_a?(Class) + + unless value.is_a?(type) + message = options[:message] || "should be a #{type.name}" + record.errors.add(attribute, message) + end + end + end + + class VariablesValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs') + end + end + end + end + end + end +end diff --git a/lib/gitlab/config/loader/format_error.rb b/lib/gitlab/config/loader/format_error.rb new file mode 100644 index 00000000000..848ff96d201 --- /dev/null +++ b/lib/gitlab/config/loader/format_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Loader + FormatError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/config/loader/yaml.rb index 141d2714cb6..8159f8b8026 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -1,13 +1,13 @@ -module Gitlab - module Ci - class Config - class Loader - FormatError = Class.new(StandardError) +# frozen_string_literal: true +module Gitlab + module Config + module Loader + class Yaml def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) rescue Psych::Exception => e - raise FormatError, e.message + raise Loader::FormatError, e.message end def valid? @@ -16,7 +16,7 @@ module Gitlab def load! unless valid? - raise FormatError, 'Invalid configuration format' + raise Loader::FormatError, 'Invalid configuration format' end @config.deep_symbolize_keys diff --git a/lib/gitlab/config_helper.rb b/lib/gitlab/config_helper.rb index 41880069e4c..b7aa03384b7 100644 --- a/lib/gitlab/config_helper.rb +++ b/lib/gitlab/config_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab::ConfigHelper def gitlab_config_features Gitlab.config.gitlab.default_projects_features diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 2a0cb640a14..0ca99506311 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,8 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Conflict class File include Gitlab::Routing include IconsHelper + include Gitlab::Utils::StrongMemoize CONTEXT_LINES = 3 @@ -30,11 +33,8 @@ module Gitlab end def highlight_lines! - their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") - our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") - - their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines - our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + their_highlight = Gitlab::Highlight.highlight(their_path, their_lines, language: their_language).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_lines, language: our_language).lines lines.each do |line| line.rich_text = @@ -158,7 +158,6 @@ module Gitlab json_hash.tap do |json_hash| if opts[:full_content] json_hash[:content] = content - json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode) else json_hash[:sections] = sections if type.text? json_hash[:type] = type @@ -183,6 +182,34 @@ module Gitlab raw_line[:line_new], parent_file: self) end end + + def their_language + strong_memoize(:their_language) do + repository.gitattribute(their_path, 'gitlab-language') + end + end + + def our_language + strong_memoize(:our_language) do + if our_path == their_path + their_language + else + repository.gitattribute(our_path, 'gitlab-language') + end + end + end + + def their_lines + strong_memoize(:their_lines) do + lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + end + end + + def our_lines + strong_memoize(:our_lines) do + lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + end + end end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 65a65b67975..53406af2c4e 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Conflict class FileCollection diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 4c28489f45a..5ed6427072a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ContributionsCalendar attr_reader :contributor @@ -7,9 +9,14 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor @current_user = current_user - @projects = ContributedProjectsFinder.new(contributor).execute(current_user) + @projects = if @contributor.include_private_contributions? + ContributedProjectsFinder.new(@contributor).execute(@contributor) + else + ContributedProjectsFinder.new(contributor).execute(current_user) + end end + # rubocop: disable CodeReuse/ActiveRecord def activity_dates return @activity_dates if @activity_dates.present? @@ -25,25 +32,25 @@ module Gitlab note_events = event_counts(date_from, :merge_requests) .having(action: [Event::COMMENTED]) - union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) - events = Event.find_by_sql(union.to_sql).map(&:attributes) + events = Event + .from_union([repo_events, issue_events, mr_events, note_events]) + .map(&:attributes) @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def events_by_date(date) return Event.none unless can_read_cross_project? - events = Event.contributions.where(author_id: contributor.id) + Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) - - # Use visible_to_user? instead of the complicated logic in activity_dates - # because we're only viewing the events for a single day. - events.select { |event| event.visible_to_user?(current_user) } end + # rubocop: enable CodeReuse/ActiveRecord def starting_year 1.year.ago.year @@ -59,13 +66,14 @@ module Gitlab Ability.allowed?(current_user, :read_cross_project) end + # rubocop: disable CodeReuse/ActiveRecord def event_counts(date_from, feature) t = Event.arel_table # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short - @contributed_project_ids ||= projects.uniq.pluck(:id) + @contributed_project_ids ||= projects.distinct.pluck(:id) authed_projects = Project.where(id: @contributed_project_ids) .with_feature_available_for_user(feature, current_user) .reorder(nil) @@ -87,5 +95,6 @@ module Gitlab .where(conditions) .where("events.project_id in (#{authed_projects.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb index c41e92b620f..d74d5a86aa0 100644 --- a/lib/gitlab/contributor.rb +++ b/lib/gitlab/contributor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class Contributor attr_accessor :email, :name, :commits, :additions, :deletions diff --git a/lib/gitlab/correlation_id.rb b/lib/gitlab/correlation_id.rb new file mode 100644 index 00000000000..0f9bde4390e --- /dev/null +++ b/lib/gitlab/correlation_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module CorrelationId + LOG_KEY = 'correlation_id'.freeze + + class << self + def use_id(correlation_id, &blk) + # always generate a id if null is passed + correlation_id ||= new_id + + ids.push(correlation_id || new_id) + + begin + yield(current_id) + ensure + ids.pop + end + end + + def current_id + ids.last + end + + def current_or_new_id + current_id || new_id + end + + private + + def ids + Thread.current[:correlation_id] ||= [] + end + + def new_id + SecureRandom.uuid + end + end + end +end diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb index 6eaed51b64c..4ddc7e02d1b 100644 --- a/lib/gitlab/cross_project_access.rb +++ b/lib/gitlab/cross_project_access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess class << self diff --git a/lib/gitlab/cross_project_access/check_collection.rb b/lib/gitlab/cross_project_access/check_collection.rb index 88376232065..55527ba5e87 100644 --- a/lib/gitlab/cross_project_access/check_collection.rb +++ b/lib/gitlab/cross_project_access/check_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess class CheckCollection diff --git a/lib/gitlab/cross_project_access/check_info.rb b/lib/gitlab/cross_project_access/check_info.rb index e8a845c7f1e..2a9eacad680 100644 --- a/lib/gitlab/cross_project_access/check_info.rb +++ b/lib/gitlab/cross_project_access/check_info.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess class CheckInfo diff --git a/lib/gitlab/cross_project_access/class_methods.rb b/lib/gitlab/cross_project_access/class_methods.rb index 90eac94800c..64ad30794d3 100644 --- a/lib/gitlab/cross_project_access/class_methods.rb +++ b/lib/gitlab/cross_project_access/class_methods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess module ClassMethods diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb new file mode 100644 index 00000000000..87a03d9c58f --- /dev/null +++ b/lib/gitlab/crypto_helper.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CryptoHelper + extend self + + AES256_GCM_OPTIONS = { + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + iv: Settings.attr_encrypted_db_key_base_12 + }.freeze + + def sha256(value) + salt = Settings.attr_encrypted_db_key_base_truncated + ::Digest::SHA256.base64digest("#{value}#{salt}") + end + + def aes256_gcm_encrypt(value) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) + Base64.strict_encode64(encrypted_token) + end + + def aes256_gcm_decrypt(value) + return unless value + + encrypted_token = Base64.decode64(value) + Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token)) + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 6cf7aa1bf0d..552aad83dd4 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,16 +1,14 @@ +# frozen_string_literal: true + module Gitlab module CurrentSettings class << self def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! - end + Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end - def fake_application_settings(attributes = {}) - Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) + def clear_in_memory_application_settings! + @in_memory_application_settings = nil end def method_missing(name, *args, &block) @@ -24,7 +22,22 @@ module Gitlab private def ensure_application_settings! + cached_application_settings || uncached_application_settings + end + + def cached_application_settings return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + + begin + ::ApplicationSetting.cached + rescue + # In case Redis isn't running + # or the Redis UNIX socket file is not available + # or the DB is not running (we use migrations in the cache key) + end + end + + def uncached_application_settings return fake_application_settings unless connect_to_db? current_settings = ::ApplicationSetting.current @@ -33,28 +46,21 @@ module Gitlab # and other callers from failing, use any loaded settings and return # defaults for missing columns. if ActiveRecord::Migrator.needs_migration? - return fake_application_settings(current_settings&.attributes) - end - - return current_settings if current_settings.present? - - with_fallback_to_fake_application_settings do - ::ApplicationSetting.create_from_defaults || in_memory_application_settings + db_attributes = current_settings&.attributes || {} + ::ApplicationSetting.build_from_defaults(db_attributes) + elsif current_settings.present? + current_settings + else + ::ApplicationSetting.create_from_defaults end end - def in_memory_application_settings - with_fallback_to_fake_application_settings do - @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults # rubocop:disable Gitlab/ModuleWithInstanceVariables - end + def fake_application_settings(attributes = {}) + Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end - def with_fallback_to_fake_application_settings(&block) - yield - rescue - # In case the application_settings table is not created yet, or if a new - # ApplicationSetting column is not yet migrated we fallback to a simple OpenStruct - fake_application_settings + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.build_from_defaults end def connect_to_db? diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index e3e3767cc75..304d60996a6 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 86d708be0d6..36231b187cd 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module BaseQuery diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 038d5a19bc4..e2d6a301734 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class BaseStage diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 06357c9b377..591db3c35e6 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class CodeEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 5f9dc9a4303..2e5f9ef5a40 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class CodeStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb index 50e126cf00b..98a30a8fc97 100644 --- a/lib/gitlab/cycle_analytics/event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module EventFetcher diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 1754f91dccb..30c6ead8968 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class IssueEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 7b03811efb2..4eae2da512c 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class IssueStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index f5d08c0b658..3e0302d308d 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module MetricsTables diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index 1e11e84a9cb..afefd09b614 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class Permissions diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 086203b9ccc..db8ac3becea 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 1a0afb56b4f..513e4575be0 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class PlanStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 0fa2e87f673..6681cb42c90 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ProductionEventFetcher < IssueEventFetcher diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index d0ca62e46e4..aff65b150fb 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module ProductionHelper diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 0fa8a65cb99..6fd7214dce7 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ProductionStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index dada819a2a8..de100295281 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ReviewEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index cfbbdc43fd9..294b656bc55 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ReviewStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb index 28e0455df59..1bd40a7aa18 100644 --- a/lib/gitlab/cycle_analytics/stage.rb +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Stage diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index fc77bd86097..5198dd5b4eb 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StageSummary diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index 2f014153ca5..70ce82383b3 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StagingEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index d5684bb9201..dbc2414ff66 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StagingStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index a917ddccac7..709221c648e 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 0a88e052f60..f0019b26fa2 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary @@ -7,9 +9,7 @@ module Gitlab end def value - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - @value ||= count_commits - end + @value ||= count_commits end private @@ -21,19 +21,11 @@ module Gitlab def count_commits return unless ref - repository = @project.repository.raw_repository - sha = @project.repository.commit(ref).sha - - cmd = %W(git --git-dir=#{repository.path} log) - cmd << '--format=%H' - cmd << "--after=#{@from.iso8601}" - cmd << sha - - output, status = Gitlab::Popen.popen(cmd) - - raise IOError, output unless status.zero? + gitaly_commit_client.commit_count(ref, after: @from) + end - output.lines.count + def gitaly_commit_client + Gitlab::GitalyClient::CommitService.new(@project.repository.raw_repository) end def ref diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 099d798aac6..3b56dc2a7bc 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 9bbf7a2685f..51695c86192 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb index a2589c6601a..4d5ea5b7c34 100644 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class TestEventFetcher < StagingEventFetcher diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index 0e9d235ca79..c31b664148b 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class TestStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb index 953268ebd46..c642809a792 100644 --- a/lib/gitlab/cycle_analytics/updater.rb +++ b/lib/gitlab/cycle_analytics/updater.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class Updater diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index 5122e3417ca..913ee373f54 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class UsageData diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index bd14c7eece3..6d5fc4219fb 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class Daemon def self.initialize_instance(*args) diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 2f1445a050a..3407380127e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Build @@ -28,6 +30,7 @@ module Gitlab build_finished_at: build.finished_at, build_duration: build.duration, build_allow_failure: build.allow_failure, + build_failure_reason: build.failure_reason, # TODO: do we still need it? project_id: project.id, diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index f573368e572..65601dcdf31 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Note diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 1e283cc092b..76c8b4ec5c2 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Pipeline @@ -26,7 +28,8 @@ module Gitlab stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, - duration: pipeline.duration + duration: pipeline.duration, + variables: pipeline.variables.map(&:hook_attrs) } end @@ -55,7 +58,7 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.is_shared? + is_shared: runner.instance_type? } end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index c169c8fe135..ea08b5f7eae 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Push @@ -29,7 +31,11 @@ module Gitlab } } ], - total_commits_count: 1 + total_commits_count: 1, + push_options: [ + "ci.skip", + "custom option" + ] }.freeze # Produce a hash of post-receive data @@ -50,10 +56,12 @@ module Gitlab # homepage: String, # }, # commits: Array, - # total_commits_count: Fixnum + # total_commits_count: Fixnum, + # push_options: Array # } # - def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil) + # rubocop:disable Metrics/ParameterLists + def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: []) commits = Array(commits) # Total commits count @@ -85,23 +93,28 @@ module Gitlab user_id: user.id, user_name: user.name, user_username: user.username, - user_email: user.email, + user_email: user.public_email, user_avatar: user.avatar_url(only_path: false), project_id: project.id, project: project.hook_attrs, commits: commit_attrs, total_commits_count: commits_count, + push_options: push_options, # DEPRECATED repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } end - # This method provide a sample data generated with + # This method provides a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) + # Use sample data if repo has no commit + # (expect the case of test service configuration settings) + return sample_data if project.empty_repo? + ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + commits = project.repository.commits(project.default_branch.to_s, limit: 3) build(project, user, commits.last&.id, commits.first&.id, ref, commits) end diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb index c9c13ec6487..0e627fd623e 100644 --- a/lib/gitlab/data_builder/repository.rb +++ b/lib/gitlab/data_builder/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Repository diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb index 226974b698c..9368446fa59 100644 --- a/lib/gitlab/data_builder/wiki_page.rb +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module WikiPage diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d49d055c3f2..b6ca777e029 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database # The max value of INTEGER type is the same between MySQL and PostgreSQL: @@ -33,7 +35,6 @@ module Gitlab adapter_name.casecmp('postgresql').zero? end - # Overridden in EE def self.read_only? false end @@ -42,10 +43,31 @@ module Gitlab !self.read_only? end + # Check whether the underlying database is in read-only mode + def self.db_read_only? + if postgresql? + pg_is_in_recovery = + ActiveRecord::Base.connection.execute('SELECT pg_is_in_recovery()') + .first.fetch('pg_is_in_recovery') + + Gitlab::Utils.to_boolean(pg_is_in_recovery) + else + false + end + end + + def self.db_read_write? + !self.db_read_only? + end + def self.version @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end + def self.postgresql_9_or_less? + postgresql? && version.to_f < 10 + end + def self.join_lateral_supported? postgresql? && version.to_f >= 9.3 end @@ -54,15 +76,37 @@ module Gitlab postgresql? && version.to_f >= 9.4 end + def self.pg_stat_wal_receiver_supported? + postgresql? && version.to_f >= 9.6 + end + + # map some of the function names that changed between PostgreSQL 9 and 10 + # https://wiki.postgresql.org/wiki/New_in_postgres_10 + def self.pg_wal_lsn_diff + Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff' + end + + def self.pg_current_wal_insert_lsn + Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn' + end + + def self.pg_last_wal_receive_lsn + Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn' + end + + def self.pg_last_wal_replay_lsn + Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn' + end + def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" if postgresql? - order << ' NULLS LAST' + order = "#{order} NULLS LAST" else # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL # columns. In the (default) ascending order, `0` comes first. - order.prepend("#{field} IS NULL, ") if direction == 'ASC' + order = "#{field} IS NULL, #{order}" if direction == 'ASC' end order @@ -72,11 +116,11 @@ module Gitlab order = "#{field} #{direction}" if postgresql? - order << ' NULLS FIRST' + order = "#{order} NULLS FIRST" else # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL # columns. In the (default) ascending order, `0` comes first. - order.prepend("#{field} IS NULL, ") if direction == 'DESC' + order = "#{field} IS NULL, #{order}" if direction == 'DESC' end order @@ -143,7 +187,7 @@ module Gitlab EOF if return_ids - sql << 'RETURNING id' + sql = "#{sql}RETURNING id" end result = connection.execute(sql) @@ -188,8 +232,7 @@ module Gitlab end def self.cached_table_exists?(table_name) - # Rails 5 uses data_source_exists? instead of table_exists? - connection.schema_cache.table_exists?(table_name) + connection.schema_cache.data_source_exists?(table_name) end private_class_method :connection @@ -205,5 +248,21 @@ module Gitlab end private_class_method :database_version + + def self.add_post_migrate_path_to_rails(force: false) + return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force + + Rails.application.config.paths['db'].each do |db_path| + path = Rails.root.join(db_path, 'post_migrate').to_s + + unless Rails.application.config.paths['db/migrate'].include? path + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path + end + end + end end end diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb deleted file mode 100644 index d7e3ce08b32..00000000000 --- a/lib/gitlab/database/arel_methods.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Gitlab - module Database - module ArelMethods - private - - # In Arel 7.0.0 (Arel 7.1.4 is used in Rails 5.0) the `engine` parameter of `Arel::UpdateManager#initializer` - # was removed. - # Remove this file and inline this method when removing rails5? code. - def arel_update_manager - if Gitlab.rails5? - Arel::UpdateManager.new - else - Arel::UpdateManager.new(ActiveRecord::Base) - end - end - end - end -end diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index 5f549ed2b3c..f3d37ccd72a 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -1,5 +1,12 @@ +# frozen_string_literal: true + # For large tables, PostgreSQL can take a long time to count rows due to MVCC. -# We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# We can optimize this by using various strategies for approximate counting. +# +# For example, we can use the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. +# +# However, since statistics are not always up to date, we also implement a table sampling strategy +# that performs an exact count but only on a sample of the table. See TablesampleCountStrategy. module Gitlab module Database module Count @@ -18,68 +25,30 @@ module Gitlab end # Takes in an array of models and returns a Hash for the approximate - # counts for them. If the model's table has not been vacuumed or - # analyzed recently, simply run the Model.count to get the data. + # counts for them. + # + # Various count strategies can be specified that are executed in + # sequence until all tables have an approximate count attached + # or we run out of strategies. + # + # Note that not all strategies are available on all supported RDBMS. # # @param [Array] # @return [Hash] of Model -> count mapping - def self.approximate_counts(models) - table_to_model_map = models.each_with_object({}) do |model, hash| - hash[model.table_name] = model - end - - table_names = table_to_model_map.keys - counts_by_table_name = Gitlab::Database.postgresql? ? reltuples_from_recently_updated(table_names) : {} - - # Convert table -> count to Model -> count - counts_by_model = counts_by_table_name.each_with_object({}) do |pair, hash| - model = table_to_model_map[pair.first] - hash[model] = pair.second - end - - missing_tables = table_names - counts_by_table_name.keys + def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) + strategies.each_with_object({}) do |strategy, counts_by_model| + if strategy.enabled? + models_with_missing_counts = models - counts_by_model.keys - missing_tables.each do |table| - model = table_to_model_map[table] - counts_by_model[model] = model.count - end - - counts_by_model - end + break counts_by_model if models_with_missing_counts.empty? - # Returns a hash of the table names that have recently updated tuples. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.reltuples_from_recently_updated(table_names) - query = postgresql_estimate_query(table_names) - rows = [] + counts = strategy.new(models_with_missing_counts).count - # Querying tuple stats only works on the primary. Due to load - # balancing, we need to ensure this query hits the load balancer. The - # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do - rows = ActiveRecord::Base.connection.select_all(query) + counts.each do |model, count| + counts_by_model[model] = count + end + end end - - rows.each_with_object({}) { |row, data| data[row['table_name']] = row['estimate'].to_i } - rescue *CONNECTION_ERRORS - {} - end - - # Generates the PostgreSQL query to return the tuples for tables - # that have been vacuumed or analyzed in the last hour. - # - # @param [Array] table names - # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) - def self.postgresql_estimate_query(table_names) - time = "to_timestamp(#{1.hour.ago.to_i})" - <<~SQL - SELECT pg_class.relname AS table_name, reltuples::bigint AS estimate FROM pg_class - LEFT JOIN pg_stat_user_tables ON pg_class.relname = pg_stat_user_tables.relname - WHERE pg_class.relname IN (#{table_names.map { |table| "'#{table}'" }.join(',')}) - AND (last_vacuum > #{time} OR last_autovacuum > #{time} OR last_analyze > #{time} OR last_autoanalyze > #{time}) - SQL end end end diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb new file mode 100644 index 00000000000..fa6951eda22 --- /dev/null +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # This strategy performs an exact count on the model. + # + # This is guaranteed to be accurate, however it also scans the + # whole table. Hence, there are no guarantees with respect + # to runtime. + # + # Note that for very large tables, this may even timeout. + class ExactCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + def count + models.each_with_object({}) do |model, data| + data[model] = model.count + end + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + true + end + end + end + end +end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb new file mode 100644 index 00000000000..c3a674aeb7e --- /dev/null +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + class PgClass < ActiveRecord::Base + self.table_name = 'pg_class' + end + + # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables. + # + # Specifically, it relies on the column reltuples in said table. An additional + # check is performed to make sure statistics were updated within the last hour. + # + # Otherwise, this strategy skips tables with outdated statistics. + # + # There are no guarantees with respect to the accuracy of this strategy. Runtime + # however is guaranteed to be "fast", because it only looks up statistics. + class ReltuplesCountStrategy + attr_reader :models + def initialize(models) + @models = models + end + + # Returns a hash of the table names that have recently updated tuples. + # + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def count + size_estimates + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? + end + + private + + def table_names + models.map(&:table_name) + end + + def size_estimates(check_statistics: true) + table_to_model = models.each_with_object({}) { |model, h| h[model.table_name] = model } + + # Querying tuple stats only works on the primary. Due to load balancing, the + # easiest way to do this is to start a transaction. + ActiveRecord::Base.transaction do + get_statistics(table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| + model = table_to_model[row.table_name] + data[model] = row.estimate + end + end + end + + # Generates the PostgreSQL query to return the tuples for tables + # that have been vacuumed or analyzed in the last hour. + # + # @param [Array] table names + # @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 }) + def get_statistics(table_names, check_statistics: true) + time = 1.hour.ago + + query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + .where(relname: table_names) + .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') + + if check_statistics + query = query.where('last_vacuum > ? OR last_autovacuum > ? OR last_analyze > ? OR last_autoanalyze > ?', + time, time, time, time) + end + + query + end + end + end + end +end diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb new file mode 100644 index 00000000000..cf1cf054dbf --- /dev/null +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Count + # A tablesample count executes in two phases: + # * Estimate table sizes based on reltuples. + # * Based on the estimate: + # * If the table is considered 'small', execute an exact relation count. + # * Otherwise, count on a sample of the table using TABLESAMPLE. + # + # The size of the sample is chosen in a way that we always roughly scan + # the same amount of rows (see TABLESAMPLE_ROW_TARGET). + # + # There are no guarantees with respect to the accuracy of the result or runtime. + class TablesampleCountStrategy < ReltuplesCountStrategy + EXACT_COUNT_THRESHOLD = 10_000 + TABLESAMPLE_ROW_TARGET = 10_000 + + def count + estimates = size_estimates(check_statistics: false) + + models.each_with_object({}) do |model, count_by_model| + count = perform_count(model, estimates[model]) + count_by_model[model] = count if count + end + rescue *CONNECTION_ERRORS + {} + end + + def self.enabled? + Gitlab::Database.postgresql? && Feature.enabled?(:tablesample_counts) + end + + private + + def perform_count(model, estimate) + # If we estimate 0, we may not have statistics at all. Don't use them. + return nil unless estimate && estimate > 0 + + if estimate < EXACT_COUNT_THRESHOLD + # The table is considered small, the assumption here is that + # the exact count will be fast anyways. + model.count + else + # The table is considered large, let's only count on a sample. + tablesample_count(model, estimate) + end + end + + def tablesample_count(model, estimate) + portion = (TABLESAMPLE_ROW_TARGET.to_f / estimate).round(4) + inverse = 1 / portion + query = <<~SQL + SELECT (COUNT(*)*#{inverse})::integer AS count + FROM #{model.table_name} TABLESAMPLE SYSTEM (#{portion * 100}) + SQL + + rows = ActiveRecord::Base.connection.select_all(query) + + Integer(rows.first['count']) + end + end + end + end +end diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 25e56998038..79d2caff151 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module DateTime diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index d32837f5793..862ab96c887 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Database # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base + include FromUnion + self.table_name = if Database.postgresql? 'information_schema.role_table_grants' @@ -42,9 +46,7 @@ module Gitlab .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')") ] - union = SQL::Union.new(queries).to_sql - - Grant.from("(#{union}) privs").any? + Grant.from_union(queries, alias_as: 'privs').any? end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 74fed447289..1455e410d4b 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # https://www.periscopedata.com/blog/medians-in-sql.html module Gitlab module Database @@ -33,7 +35,7 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table + query = arel_table.from .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) .project(average([arel_table[column_sym]], 'median')) .where( @@ -143,7 +145,7 @@ module Gitlab .order(arel_table[column_sym]) ).as('row_id') - count = arel_table.from(arel_table.alias) + count = arel_table.from.from(arel_table.alias) .project('COUNT(*)') .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) .as('ct') diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index c21bae5e16b..3abd0600e9d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,8 +1,8 @@ +# frozen_string_literal: true + module Gitlab module Database module MigrationHelpers - include Gitlab::Database::ArelMethods - BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time @@ -58,7 +58,6 @@ module Gitlab if Database.postgresql? options = options.merge({ algorithm: :concurrently }) - disable_statement_timeout end if index_exists?(table_name, column_name, options) @@ -66,7 +65,9 @@ module Gitlab return end - add_index(table_name, column_name, options) + disable_statement_timeout do + add_index(table_name, column_name, options) + end end # Removes an existed index, concurrently when supported @@ -87,7 +88,6 @@ module Gitlab if supports_drop_index_concurrently? options = options.merge({ algorithm: :concurrently }) - disable_statement_timeout end unless index_exists?(table_name, column_name, options) @@ -95,7 +95,9 @@ module Gitlab return end - remove_index(table_name, options.merge({ column: column_name })) + disable_statement_timeout do + remove_index(table_name, options.merge({ column: column_name })) + end end # Removes an existing index, concurrently when supported @@ -116,7 +118,6 @@ module Gitlab if supports_drop_index_concurrently? options = options.merge({ algorithm: :concurrently }) - disable_statement_timeout end unless index_exists_by_name?(table_name, index_name) @@ -124,7 +125,9 @@ module Gitlab return end - remove_index(table_name, options.merge({ name: index_name })) + disable_statement_timeout do + remove_index(table_name, options.merge({ name: index_name })) + end end # Only available on Postgresql >= 9.2 @@ -171,8 +174,6 @@ module Gitlab on_delete = 'SET NULL' if on_delete == :nullify end - disable_statement_timeout - key_name = concurrent_foreign_key_name(source, column) unless foreign_key_exists?(source, target, column: column) @@ -199,7 +200,9 @@ module Gitlab # while running. # # Note this is a no-op in case the constraint is VALID already - execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") + disable_statement_timeout do + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") + end end def foreign_key_exists?(source, target = nil, column: nil) @@ -224,8 +227,48 @@ module Gitlab # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure # migrations don't get killed prematurely. (PostgreSQL only) + # + # There are two possible ways to disable the statement timeout: + # + # - Per transaction (this is the preferred and default mode) + # - Per connection (requires a cleanup after the execution) + # + # When using a per connection disable statement, code must be inside + # a block so we can automatically execute `RESET ALL` after block finishes + # otherwise the statement will still be disabled until connection is dropped + # or `RESET ALL` is executed def disable_statement_timeout - execute('SET statement_timeout TO 0') if Database.postgresql? + # bypass disabled_statement logic when not using postgres, but still execute block when one is given + unless Database.postgresql? + if block_given? + yield + end + + return + end + + if block_given? + begin + execute('SET statement_timeout TO 0') + + yield + ensure + execute('RESET ALL') + end + else + unless transaction_open? + raise <<~ERROR + Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block. + If you don't want to use a transaction wrap your code in a block call: + + disable_statement_timeout { # code that requires disabled statement here } + + This will make sure statement_timeout is disabled before and reset after the block execution is finished. + ERROR + end + + execute('SET LOCAL statement_timeout TO 0') + end end def true_value @@ -316,7 +359,7 @@ module Gitlab stop_arel = yield table, stop_arel if block_given? stop_row = exec_query(stop_arel.to_sql).to_hash.first - update_arel = arel_update_manager + update_arel = Arel::UpdateManager.new .table(table) .set([[table[column], value]]) .where(table[:id].gteq(start_id)) @@ -367,30 +410,30 @@ module Gitlab 'in the body of your migration class' end - disable_statement_timeout - - transaction do - if limit - add_column(table, column, type, default: nil, limit: limit) - else - add_column(table, column, type, default: nil) + disable_statement_timeout do + transaction do + if limit + add_column(table, column, type, default: nil, limit: limit) + else + add_column(table, column, type, default: nil) + end + + # Changing the default before the update ensures any newly inserted + # rows already use the proper default value. + change_column_default(table, column, default) end - # Changing the default before the update ensures any newly inserted - # rows already use the proper default value. - change_column_default(table, column, default) - end - - begin - update_column_in_batches(table, column, default, &block) + begin + update_column_in_batches(table, column, default, &block) - change_column_null(table, column, false) unless allow_null - # We want to rescue _all_ exceptions here, even those that don't inherit - # from StandardError. - rescue Exception => error # rubocop: disable all - remove_column(table, column) + change_column_null(table, column, false) unless allow_null + # We want to rescue _all_ exceptions here, even those that don't inherit + # from StandardError. + rescue Exception => error # rubocop: disable all + remove_column(table, column) - raise error + raise error + end end end @@ -596,6 +639,97 @@ module Gitlab end end + # Renames a column using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `rename_column_concurrently` since it can complete its work in a much + # shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # rename_column_using_background_migration( + # :users, + # :feed_token, + # :rss_token + # ) + # + # table - The name of the database table containing the column. + # + # old - The old column name. + # + # new - The new column name. + # + # type - The type of the new column. If no type is given the old column's + # type is used. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def rename_column_using_background_migration( + table, + old_column, + new_column, + type: nil, + batch_size: 10_000, + interval: 10.minutes + ) + + check_trigger_permissions!(table) + + old_col = column_for(table, old_column) + new_type = type || old_col.type + max_index = 0 + + add_column(table, new_column, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new_column, old_col.default) if old_col.default + + install_rename_triggers(table, old_column, new_column) + + model = Class.new(ActiveRecord::Base) do + self.table_name = table + + include ::EachBatch + end + + # Schedule the jobs that will copy the data from the old column to the + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, old_column, new_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentRename', + [table, old_column, new_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') + end + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end @@ -745,7 +879,7 @@ module Gitlab columns(table).find { |column| column.name == name } end - # This will replace the first occurance of a string in a column with + # This will replace the first occurrence of a string in a column with # the replacement # On postgresql we can use `regexp_replace` for that. # On mysql we find the location of the pattern, and overwrite it @@ -803,7 +937,7 @@ database (#{dbname}) using a super user and running: For MySQL you instead need to run: - GRANT ALL PRIVILEGES ON *.* TO #{user}@'%' + GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%' Both queries will grant the user super user permissions, ensuring you don't run into similar problems in the future (e.g. when new tables are created). @@ -839,9 +973,10 @@ into similar problems in the future (e.g. when new tables are created). raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') jobs = [] + table_name = model_class.quoted_table_name model_class.each_batch(of: batch_size) do |relation| - start_id, end_id = relation.pluck('MIN(id), MAX(id)').first + start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE # Note: This code path generally only helps with many millions of rows @@ -888,8 +1023,8 @@ into similar problems in the future (e.g. when new tables are created). # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL - delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + if delay_interval < BackgroundMigrationWorker.minimum_interval + delay_interval = BackgroundMigrationWorker.minimum_interval end model_class.each_batch(of: batch_size) do |relation, index| @@ -939,6 +1074,10 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(index_sql).to_i > 0 end + + def mysql_compatible_index_length + Gitlab::Database.mysql? ? 20 : nil + end end end end diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb index 7ae5a4c17c8..1d39a3d0b57 100644 --- a/lib/gitlab/database/multi_threaded_migration.rb +++ b/lib/gitlab/database/multi_threaded_migration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module MultiThreadedMigration diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb index 4571ad122ce..2362208e5dd 100644 --- a/lib/gitlab/database/read_only_relation.rb +++ b/lib/gitlab/database/read_only_relation.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database # Module that can be injected into a ActiveRecord::Relation to make it diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb index f333ff22300..2314246da55 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This module can be included in migrations to make it easier to rename paths # of `Namespace` & `Project` models certain paths would become `reserved`. # diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 62d4d0a92a6..f1dc3ed74fe 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration @@ -37,6 +39,7 @@ module Gitlab class Namespace < ActiveRecord::Base include MigrationClasses::Routable self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled belongs_to :parent, class_name: "#{MigrationClasses.name}::Namespace" has_one :route, as: :source diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 14de28a1d08..60afa4bcd52 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -1,10 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration module V1 class RenameBase - include Gitlab::Database::ArelMethods - attr_reader :paths, :migration delegate :update_column_in_batches, @@ -64,7 +64,7 @@ module Gitlab old_full_path, new_full_path) - update = arel_update_manager + update = Arel::UpdateManager.new .table(routes) .set([[routes[:path], replace_statement]]) .where(Arel::Nodes::SqlLiteral.new(filter)) diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 73971af6a74..6bbad707f0f 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 827aeb12a02..580be9fe267 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index b2d8ee81977..8d97adaff99 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database BINARY_TYPE = @@ -6,14 +8,7 @@ module Gitlab # behaviour from the default Binary type. ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea else - # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` - # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b - # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. - if Gitlab.rails5? - ActiveModel::Type::Binary - else - ActiveRecord::Type::Binary - end + ActiveModel::Type::Binary end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). @@ -24,31 +19,9 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # It is called from activerecord-4.2.10/lib/active_record internal methods. - # Remove this method when removing Gitlab.rails5? code. - def type_cast_from_database(value) - unpack_sha(super) - end - - # It is called from activerecord-4.2.10/lib/active_record internal methods. - # Remove this method when removing Gitlab.rails5? code. - def type_cast_for_database(value) - serialize(value) - end - - # It is called from activerecord-5.0.6/lib/active_record/attribute.rb - # Remove this method when removing Gitlab.rails5? code.. - def deserialize(value) - value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) - - unpack_sha(value) - end - - # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. # Casts binary data to a SHA1 in hexadecimal. - def unpack_sha(value) - # Uncomment this line when removing Gitlab.rails5? code. - # value = super + def deserialize(value) + value = super(value) value ? value.unpack(PACK_FORMAT)[0] : nil end @@ -56,7 +29,7 @@ module Gitlab def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) + super(arg) end end end diff --git a/lib/gitlab/database/subquery.rb b/lib/gitlab/database/subquery.rb new file mode 100644 index 00000000000..10971d2b274 --- /dev/null +++ b/lib/gitlab/database/subquery.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Subquery + class << self + def self_join(relation) + t = relation.arel_table + # Work around a bug in Rails 5, where LIMIT causes trouble + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/51729 + r = relation.limit(nil).arel + r.take(relation.limit_value) if relation.limit_value + t2 = r.as('t2') + + relation.unscoped.joins(t.join(t2).on(t[:id].eq(t2[:id])).join_sources.first) + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index 3192bf6f667..c63d9e5bb71 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker LINKERS = [ diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index d2360583741..ac2efe598b4 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class BaseLinker diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb index 4f69f2c4ab2..0e33f0956dd 100644 --- a/lib/gitlab/dependency_linker/cartfile_linker.rb +++ b/lib/gitlab/dependency_linker/cartfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class CartfileLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb index 2fbde7da1b4..38eabe303de 100644 --- a/lib/gitlab/dependency_linker/cocoapods.rb +++ b/lib/gitlab/dependency_linker/cocoapods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker module Cocoapods diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index cfd4ec15125..22d2bead891 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class ComposerJsonLinker < PackageJsonLinker diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index bfea836bcb2..8ab219c4962 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GemfileLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb index f1783ee2ab4..b924ea86d89 100644 --- a/lib/gitlab/dependency_linker/gemspec_linker.rb +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GemspecLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb index fe091baee6d..d24c137793e 100644 --- a/lib/gitlab/dependency_linker/godeps_json_linker.rb +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GodepsJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb index a8ef25233d8..298d214df61 100644 --- a/lib/gitlab/dependency_linker/json_linker.rb +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class JsonLinker < BaseLinker diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb index 0ffa2a83c93..d4d85bb3390 100644 --- a/lib/gitlab/dependency_linker/method_linker.rb +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class MethodLinker < BaseLinker diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb index 330c95f0880..578e25f806a 100644 --- a/lib/gitlab/dependency_linker/package_json_linker.rb +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PackageJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb index 60ad166ea17..def9b04cca9 100644 --- a/lib/gitlab/dependency_linker/podfile_linker.rb +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodfileLinker < GemfileLinker diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb index d82237ed3f1..1a2493e7cc0 100644 --- a/lib/gitlab/dependency_linker/podspec_json_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodspecJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index 924e55e9820..6b1758c5a43 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodspecLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index 9c9620bc36a..f630c13b760 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class RequirementsTxtLinker < BaseLinker diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 81df47964be..dc245377ccc 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class DiffRefs @@ -21,7 +23,7 @@ module Gitlab alias_method :eql?, :== def hash - [base_sha, start_sha, head_sha].hash + [self.class, base_sha, start_sha, head_sha].hash end # There is only one case in which we will have `start_sha` and `head_sha`, diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 765fb0289a8..e410d5a8333 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier delegate :new_file?, :deleted_file?, :renamed_file?, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, @@ -20,11 +24,21 @@ module Gitlab DiffViewer::Image ].sort_by { |v| v.binary? ? 0 : 1 }.freeze - def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) + def initialize( + diff, + repository:, + diff_refs: nil, + fallback_diff_refs: nil, + stats: nil, + unique_identifier: nil) + @diff = diff + @stats = stats @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @unique_identifier = unique_identifier + @unfolded = false # Ensure items are collected in the the batch new_blob_lazy @@ -63,7 +77,15 @@ module Gitlab def line_for_position(pos) return nil unless pos.position_type == 'text' - diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } + # This method is normally used to find which line the diff was + # commented on, and in this context, it's normally the raw diff persisted + # at `note_diff_files`, which is a fraction of the entire diff + # (it goes from the first line, to the commented line, or + # one line below). Therefore it's more performant to fetch + # from bottom to top instead of the other way around. + diff_lines + .reverse_each + .find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } end def position_for_line_code(code) @@ -78,9 +100,12 @@ module Gitlab # Returns the raw diff content up to the given line index def diff_hunk(diff_line) - # Adding 2 because of the @@ diff header and Enum#take should consider - # an extra line, because we're passing an index. - raw_diff.each_line.take(diff_line.index + 2).join + diff_line_index = diff_line.index + # @@ (match) header is not kept if it's found in the top of the file, + # therefore we should keep an extra line on this scenario. + diff_line_index += 1 unless diff_lines.first.match? + + diff_lines.select { |line| line.index <= diff_line_index }.map(&:text).join("\n") end def old_sha @@ -115,6 +140,16 @@ module Gitlab old_blob_lazy&.itself end + def new_blob_lines_between(from_line, to_line) + return [] unless new_blob + + from_index = from_line - 1 + to_index = to_line - 1 + + new_blob.load_all_data! + new_blob.data.lines[from_index..to_index] + end + def content_sha new_content_sha || old_content_sha end @@ -127,11 +162,35 @@ module Gitlab # Array of Gitlab::Diff::Line objects def diff_lines - @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + @diff_lines ||= + Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a + end + + # Changes diff_lines according to the given position. That is, + # it checks whether the position requires blob lines into the diff + # in order to be presented. + def unfold_diff_lines(position) + return unless position + + unfolder = Gitlab::Diff::LinesUnfolder.new(self, position) + + if unfolder.unfold_required? + @diff_lines = unfolder.unfolded_diff_lines + @unfolded = true + end + end + + def unfolded? + @unfolded + end + + def highlight_loaded? + @highlighted_diff_lines.present? end def highlighted_diff_lines - @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight + @highlighted_diff_lines ||= + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted @@ -160,11 +219,11 @@ module Gitlab end def added_lines - diff_lines.count(&:added?) + @stats&.additions || diff_lines.count(&:added?) end def removed_lines - diff_lines.count(&:removed?) + @stats&.deletions || diff_lines.count(&:removed?) end def file_identifier @@ -175,12 +234,12 @@ module Gitlab repository.attributes(file_path).fetch('diff') { true } end - def binary? - has_binary_notice? || try_blobs(:binary?) + def binary_in_repo? + has_binary_notice? || try_blobs(:binary_in_repo?) end - def text? - !binary? + def text_in_repo? + !binary_in_repo? end def external_storage_error? @@ -206,20 +265,32 @@ module Gitlab old_blob && new_blob && old_blob.binary? != new_blob.binary? end + # rubocop: disable CodeReuse/ActiveRecord def size valid_blobs.map(&:size).sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_size valid_blobs.map(&:raw_size).sum end + # rubocop: enable CodeReuse/ActiveRecord - def raw_binary? - try_blobs(:raw_binary?) + def empty? + valid_blobs.map(&:empty?).all? end - def raw_text? - !raw_binary? && !different_type? + def binary? + strong_memoize(:is_binary) do + try_blobs(:binary?) + end + end + + def text? + strong_memoize(:is_text) do + !binary? && !different_type? + end end def simple_viewer @@ -236,8 +307,34 @@ module Gitlab simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) end + # This adds the bottom match line to the array if needed. It contains + # the data to load more context lines. + def diff_lines_for_serializer + lines = highlighted_diff_lines + + return if lines.empty? + return if blob.nil? + + last_line = lines.last + + if last_line.new_pos < total_blob_lines(blob) && !deleted_file? + match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) + lines.push(match_line) + end + + lines + end + private + def total_blob_lines(blob) + @total_lines ||= begin + line_count = blob.lines.size + line_count -= 1 if line_count > 0 && blob.lines.last.blank? + line_count + end + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) @@ -276,19 +373,19 @@ module Gitlab return DiffViewer::NotDiffable unless diffable? if content_changed? - if raw_text? + if text? DiffViewer::Text else DiffViewer::NoPreview end elsif new_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Added end elsif deleted_file? - if raw_text? + if text? DiffViewer::Text else DiffViewer::Deleted diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index c79d8d3cb21..c5bbf522f7c 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -1,28 +1,47 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) @diffable = diffable - @diffs = diffable.raw_diffs(diff_options) + @include_stats = diff_options.delete(:include_stats) @project = project @diff_options = diff_options @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @repository = project.repository + end + + def diffs + @diffs ||= diffable.raw_diffs(diff_options) end def diff_files - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } + @diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) } + end + + # This mutates `diff_files` lines. + def unfold_diff_files(positions) + positions_grouped_by_path = positions.group_by { |position| position.file_path } + + diff_files.each do |diff_file| + positions = positions_grouped_by_path.fetch(diff_file.file_path, []) + positions.each { |position| diff_file.unfold_diff_lines(position) } + end end def diff_file_with_old_path(old_path) @@ -33,12 +52,37 @@ module Gitlab diff_files.find { |diff_file| diff_file.new_path == new_path } end + def clear_cache + # No-op + end + + def write_cache + # No-op + end + private + def diff_stats_collection + strong_memoize(:diff_stats) do + # There are scenarios where we don't need to request Diff Stats, + # when caching for instance. + next unless @include_stats + next unless diff_refs + + @repository.diff_stats(diff_refs.base_sha, diff_refs.head_sha) + end + end + def decorate_diff!(diff) return diff if diff.is_a?(File) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) + stats = diff_stats_collection&.find_by_path(diff.new_path) + + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: diff_refs, + fallback_diff_refs: fallback_diff_refs, + stats: stats) end end end diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb index 4dc297ec036..7b1d6171e82 100644 --- a/lib/gitlab/diff/file_collection/commit.rb +++ b/lib/gitlab/diff/file_collection/commit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 20d8f891cc3..663bad95db7 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection @@ -8,6 +10,10 @@ module Gitlab diff_options: diff_options, diff_refs: diff_refs) end + + def unfold_diff_lines(positions) + # no-op + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index c358ae428cf..e29bf75f341 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection class MergeRequestDiff < Base + extend ::Gitlab::Utils::Override + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -13,70 +17,35 @@ module Gitlab end def diff_files - # Make sure to _not_ send any method call to Gitlab::Diff::File - # _before_ all of them were collected (`super`). Premature method calls will - # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy). - # diff_files = super - diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) } - store_highlight_cache + diff_files.each { |diff_file| cache.decorate(diff_file) } diff_files end - def real_size - @merge_request_diff.real_size + override :write_cache + def write_cache + cache.write_if_empty end - def clear_cache! - Rails.cache.delete(cache_key) + override :clear_cache + def clear_cache + cache.clear end def cache_key - [@merge_request_diff, 'highlighted-diff-files', diff_options] - end - - private - - def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) - diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| - Gitlab::Diff::Line.init_from_hash(line) - end + cache.key end - # - # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted) - # for the highlighted ones, so we just skip their execution. - # If the highlighted diff files lines are not cached we calculate and cache them. - # - # The content of the cache is a Hash where the key identifies the file and the values are Arrays of - # hashes that represent serialized diff lines. - # - def cache_highlight!(diff_file) - item_key = diff_file.file_identifier - - if highlight_cache[item_key] - highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) - else - highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) - end - end - - def highlight_cache - return @highlight_cache if defined?(@highlight_cache) - - @highlight_cache = Rails.cache.read(cache_key) || {} - @highlight_cache_was_empty = @highlight_cache.empty? - @highlight_cache + def real_size + @merge_request_diff.real_size end - def store_highlight_cache - Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty - end + private - def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.text? && diff_file.diffable? + def cache + @cache ||= Gitlab::Diff::HighlightCache.new(self) end end end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 5e923b9e602..9704aed82c1 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb index ccd0d309972..5bc9f0c337f 100644 --- a/lib/gitlab/diff/formatters/image_formatter.rb +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 01c7e9f51ab..f6e247ef665 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 5c1baa19b66..d2484217ab9 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class Highlight @@ -24,7 +26,7 @@ module Gitlab # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || diff_line.text + rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) if line_inline_diffs = inline_diffs[i] begin @@ -37,7 +39,7 @@ module Gitlab end end - diff_line.text = rich_line + diff_line.rich_text = rich_line diff_line end @@ -79,7 +81,7 @@ module Gitlab return [] unless blob blob.load_all_data! - Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines + blob.present.highlight.lines end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb new file mode 100644 index 00000000000..e4390771db2 --- /dev/null +++ b/lib/gitlab/diff/highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class HighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb index 65332dfd239..a3ce032f8e2 100644 --- a/lib/gitlab/diff/image_point.rb +++ b/lib/gitlab/diff/image_point.rb @@ -1,13 +1,15 @@ +# frozen_string_literal: true + module Gitlab module Diff class ImagePoint attr_reader :width, :height, :x, :y - def initialize(width, height, x, y) + def initialize(width, height, new_x, new_y) @width = width @height = height - @x = x - @y = y + @x = new_x + @y = new_y end def to_h diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 54783a07919..5815d1bae4a 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiff @@ -67,10 +69,11 @@ module Gitlab private # Finds pairs of old/new line pairs that represent the same line that changed + # rubocop: disable CodeReuse/ActiveRecord def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') + line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') changed_line_pairs = [] line_prefixes.scan(LINE_PAIRS_PATTERN) do @@ -89,11 +92,12 @@ module Gitlab changed_line_pairs end + # rubocop: enable CodeReuse/ActiveRecord end private - def longest_common_prefix(a, b) + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max length = 0 @@ -109,7 +113,7 @@ module Gitlab length end - def longest_common_suffix(a, b) + def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName longest_common_prefix(a.reverse, b.reverse) end end diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb index c2a2eb15931..3c536c43a9e 100644 --- a/lib/gitlab/diff/inline_diff_markdown_marker.rb +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 81e91ea0ab7..1bbde1ffd2a 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0603141e441..001748afb41 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,27 +1,39 @@ +# frozen_string_literal: true + module Gitlab module Diff class Line - attr_reader :type, :index, :old_pos, :new_pos + SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze + + attr_reader :line_code, :type, :old_pos, :new_pos attr_writer :rich_text - attr_accessor :text + attr_accessor :text, :index - def initialize(text, type, index, old_pos, new_pos, parent_file: nil) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos @parent_file = parent_file - end + @rich_text = rich_text - def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos]) + # When line code is not provided from cache store we build it + # using the parent_file(Diff::File or Conflict::File). + @line_code = line_code || calculate_line_code end - def serialize_keys - @serialize_keys ||= %i(text type index old_pos new_pos) + def self.init_from_hash(hash) + new(hash[:text], + hash[:type], + hash[:index], + hash[:old_pos], + hash[:new_pos], + parent_file: hash[:parent_file], + line_code: hash[:line_code], + rich_text: hash[:rich_text]) end def to_hash hash = {} - serialize_keys.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend + SERIALIZE_KEYS.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend hash end @@ -53,25 +65,44 @@ module Gitlab %w[match new-nonewline old-nonewline].include?(type) end + def match? + type == :match + end + def discussable? !meta? end + def suggestible? + !removed? + end + def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text + @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text @rich_text end - def as_json(opts = nil) + def meta_positions + return unless meta? + { - type: type, - old_line: old_line, - new_line: new_line, - text: text, - rich_text: rich_text || text + old_pos: old_pos, + new_pos: new_pos } end + + # We have to keep this here since it is still used for conflict resolution + # Conflict::File#as_json renders json diff lines in sections + def as_json(opts = nil) + DiffLineSerializer.new.represent(self) + end + + private + + def calculate_line_code + @parent_file&.line_code(self) + end end end end diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb index cf71d47df8e..fba7bff4781 100644 --- a/lib/gitlab/diff/line_mapper.rb +++ b/lib/gitlab/diff/line_mapper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # When provided a diff for a specific file, maps old line numbers to new line # numbers and back, to find out where a specific line in a file was moved by the # changes. diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb new file mode 100644 index 00000000000..6cf904b2b2a --- /dev/null +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -0,0 +1,240 @@ +# frozen_string_literal: true + +# Given a position, calculates which Blob lines should be extracted, treated and +# injected in the current diff file lines in order to present a "unfolded" diff. +module Gitlab + module Diff + class LinesUnfolder + include Gitlab::Utils::StrongMemoize + + UNFOLD_CONTEXT_SIZE = 3 + + def initialize(diff_file, position) + @diff_file = diff_file + @blob = diff_file.old_blob + @position = position + @generate_top_match_line = true + @generate_bottom_match_line = true + + # These methods update `@generate_top_match_line` and + # `@generate_bottom_match_line`. + @from_blob_line = calculate_from_blob_line! + @to_blob_line = calculate_to_blob_line! + end + + # Returns merged diff lines with required blob lines with correct + # positions. + def unfolded_diff_lines + strong_memoize(:unfolded_diff_lines) do + next unless unfold_required? + + merged_diff_with_blob_lines + end + end + + # Returns the extracted lines from the old blob which should be merged + # with the current diff lines. + def blob_lines + strong_memoize(:blob_lines) do + # Blob lines, unlike diffs, doesn't start with an empty space for + # unchanged line, so the parsing and highlighting step can get fuzzy + # without the following change. + line_prefix = ' ' + blob_as_diff_lines = @blob.data.each_line.map { |line| "#{line_prefix}#{line}" } + + lines = Gitlab::Diff::Parser.new.parse(blob_as_diff_lines, diff_file: @diff_file).to_a + + from = from_blob_line - 1 + to = to_blob_line - 1 + + lines[from..to] + end + end + + def unfold_required? + strong_memoize(:unfold_required) do + next false unless @diff_file.text? + next false unless @position.unchanged? + next false if @diff_file.new_file? || @diff_file.deleted_file? + next false unless @position.old_line + # Invalid position (MR import scenario) + next false if @position.old_line > @blob.lines.size + next false if @diff_file.diff_lines.empty? + next false if @diff_file.line_for_position(@position) + next false unless unfold_line + + true + end + end + + private + + attr_reader :from_blob_line, :to_blob_line + + def merged_diff_with_blob_lines + lines = @diff_file.diff_lines + match_line = unfold_line + insert_index = bottom? ? -1 : match_line.index + + lines -= [match_line] unless bottom? + + lines.insert(insert_index, *blob_lines_with_matches) + + # The inserted blob lines have invalid indexes, so we need + # to reindex them. + reindex(lines) + + lines + end + + # Returns 'unchanged' blob lines with recalculated `old_pos` and + # `new_pos` and the recalculated new match line (needed if we for instance + # we unfolded once, but there are still folded lines). + def blob_lines_with_matches + old_pos = from_blob_line + new_pos = from_blob_line + offset + + new_blob_lines = [] + + new_blob_lines.push(top_blob_match_line) if top_blob_match_line + + blob_lines.each do |line| + new_blob_lines << Gitlab::Diff::Line.new(line.text, line.type, nil, old_pos, new_pos, + parent_file: @diff_file) + + old_pos += 1 + new_pos += 1 + end + + new_blob_lines.push(bottom_blob_match_line) if bottom_blob_match_line + + new_blob_lines + end + + def reindex(lines) + lines.each_with_index { |line, i| line.index = i } + end + + def top_blob_match_line + strong_memoize(:top_blob_match_line) do + next unless @generate_top_match_line + + old_pos = from_blob_line + new_pos = from_blob_line + offset + + build_match_line(old_pos, new_pos) + end + end + + def bottom_blob_match_line + strong_memoize(:bottom_blob_match_line) do + # The bottom line match addition is already handled on + # Diff::File#diff_lines_for_serializer + next if bottom? + next unless @generate_bottom_match_line + + position = line_after_unfold_position.old_pos + + old_pos = position + new_pos = position + offset + + build_match_line(old_pos, new_pos) + end + end + + def build_match_line(old_pos, new_pos) + blob_lines_length = blob_lines.length + old_line_ref = [old_pos, blob_lines_length].join(',') + new_line_ref = [new_pos, blob_lines_length].join(',') + new_match_line_str = "@@ -#{old_line_ref}+#{new_line_ref} @@" + + Gitlab::Diff::Line.new(new_match_line_str, 'match', nil, old_pos, new_pos) + end + + # Returns the first line position that should be extracted + # from `blob_lines`. + def calculate_from_blob_line! + return unless unfold_required? + + from = comment_position - UNFOLD_CONTEXT_SIZE + + prev_line_number = + if bottom? + last_line.old_pos + else + # There's no line before the match if it's in the top-most + # position. + line_before_unfold_position&.old_pos || 0 + end + + if from <= prev_line_number + 1 + @generate_top_match_line = false + from = prev_line_number + 1 + end + + from + end + + # Returns the last line position that should be extracted + # from `blob_lines`. + def calculate_to_blob_line! + return unless unfold_required? + + to = comment_position + UNFOLD_CONTEXT_SIZE + + return to if bottom? + + next_line_number = line_after_unfold_position.old_pos + + if to >= next_line_number - 1 + @generate_bottom_match_line = false + to = next_line_number - 1 + end + + to + end + + def offset + unfold_line.new_pos - unfold_line.old_pos + end + + def line_before_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index - 1] if index > 0 + end + + def line_after_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index + 1] if index >= 0 + end + + def bottom? + strong_memoize(:bottom) do + @position.old_line > last_line.old_pos + end + end + + # Returns the line which needed to be expanded in order to send a comment + # in `@position`. + def unfold_line + strong_memoize(:unfold_line) do + next last_line if bottom? + + @diff_file.diff_lines.find do |line| + line.old_pos > comment_position && line.type == 'match' + end + end + end + + def comment_position + @position.old_line + end + + def last_line + @diff_file.diff_lines.last + end + end + end +end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 0cb26fa45c8..77b65fea726 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class ParallelDiff diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 8302f30a0a2..4a47e4b80b6 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Diff class Parser include Enumerable - def parse(lines) + def parse(lines, diff_file: nil) return [] if lines.blank? @lines = lines @@ -31,17 +33,17 @@ module Gitlab next if line_old <= 1 && line_new <= 1 # top of file - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 next elsif line[0] == '\\' type = "#{context}-nonewline" - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 else type = identification_type(line) - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 978962ab2eb..e8f98f52111 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab @@ -69,6 +71,10 @@ module Gitlab JSON.generate(formatter.to_h, opts) end + def as_json(opts = nil) + to_h.as_json(opts) + end + def type formatter.line_age end @@ -97,25 +103,29 @@ module Gitlab @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) end + def unfolded_diff?(repository) + diff_file(repository)&.unfolded? + end + def diff_file(repository) return @diff_file if defined?(@diff_file) @diff_file = begin - if RequestStore.active? - key = { - project_id: repository.project.id, - start_sha: start_sha, - head_sha: head_sha, - path: file_path - } - - RequestStore.fetch(key) { find_diff_file(repository) } - else - find_diff_file(repository) - end + key = { + project_id: repository.project.id, + start_sha: start_sha, + head_sha: head_sha, + path: file_path + } + + Gitlab::SafeRequestStore.fetch(key) { find_diff_file(repository) } end end + def diff_options + { paths: paths, expanded: true, include_stats: false } + end + def diff_line(repository) @diff_line ||= diff_file(repository)&.line_for_position(self) end @@ -130,7 +140,13 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(paths: paths, expanded: true).diff_files.first + file = comparison.diffs(diff_options).diff_files.first + + # We need to unfold diff lines according to the position in order + # to correctly calculate the line code and trace position changes. + file&.unfold_diff_lines(self) + + file end def get_formatter_class(type) diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index b68a1636814..af3df820422 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Finds the diff position in the new diff that corresponds to the same location # specified by the provided position in the old diff. module Gitlab @@ -24,7 +26,7 @@ module Gitlab # head of `feature` was commit B, resulting in the original diff A->B. # Since creation, `master` was updated to C. # Now `feature` is being updated to D, and the newly generated MR diff is C->D. - # It is possible that C and D are direct decendants of A and B respectively, + # It is possible that C and D are direct descendants of A and B respectively, # but this isn't necessarily the case as rebases and merges come into play. # # Suppose we have a diff note on the original diff A->B. Now that the MR diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb new file mode 100644 index 00000000000..4ab7314f509 --- /dev/null +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module DiscussionsDiff + class FileCollection + include Gitlab::Utils::StrongMemoize + + def initialize(collection) + @collection = collection + end + + # Returns a Gitlab::Diff::File with the given ID (`unique_identifier` in + # Gitlab::Diff::File). + def find_by_id(id) + diff_files_indexed_by_id[id] + end + + # Writes cache and preloads highlighted diff lines for + # object IDs, in @collection. + # + # highlightable_ids - Diff file `Array` responding to ID. The ID will be used + # to generate the cache key. + # + # - Highlight cache is written just for uncached diff files + # - The cache content is not updated (there's no need to do so) + def load_highlight(highlightable_ids) + preload_highlighted_lines(highlightable_ids) + end + + private + + def preload_highlighted_lines(ids) + cached_content = read_cache(ids) + + uncached_ids = ids.select.each_with_index { |_, i| cached_content[i].nil? } + mapping = highlighted_lines_by_ids(uncached_ids) + + HighlightCache.write_multiple(mapping) + + diffs = diff_files_indexed_by_id.values_at(*ids) + + diffs.zip(cached_content).each do |diff, cached_lines| + next unless diff && cached_lines + + diff.highlighted_diff_lines = cached_lines + end + end + + def read_cache(ids) + HighlightCache.read_multiple(ids) + end + + def diff_files_indexed_by_id + strong_memoize(:diff_files_indexed_by_id) do + diff_files.index_by(&:unique_identifier) + end + end + + def diff_files + strong_memoize(:diff_files) do + @collection.map(&:raw_diff_file) + end + end + + # Processes the diff lines highlighting for diff files matching the given + # IDs. + # + # Returns a Hash with { id => [Array of Gitlab::Diff::line], ...] + def highlighted_lines_by_ids(ids) + diff_files_indexed_by_id.slice(*ids).each_with_object({}) do |(id, file), hash| + hash[id] = file.highlighted_diff_lines.map(&:to_hash) + end + end + end + end +end diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb new file mode 100644 index 00000000000..270cfb89488 --- /dev/null +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +# +module Gitlab + module DiscussionsDiff + class HighlightCache + class << self + VERSION = 1 + EXPIRATION = 1.week + + # Sets multiple keys to a given value. The value + # is serialized as JSON. + # + # mapping - Write multiple cache values at once + def write_multiple(mapping) + Redis::Cache.with do |redis| + redis.multi do |multi| + mapping.each do |raw_key, value| + key = cache_key_for(raw_key) + + multi.set(key, value.to_json, ex: EXPIRATION) + end + end + end + end + + # Reads multiple cache keys at once. + # + # raw_keys - An Array of unique cache keys, without namespaces. + # + # It returns a list of deserialized diff lines. Ex.: + # [[Gitlab::Diff::Line, ...], [Gitlab::Diff::Line]] + def read_multiple(raw_keys) + return [] if raw_keys.empty? + + keys = raw_keys.map { |id| cache_key_for(id) } + + content = + Redis::Cache.with do |redis| + redis.mget(keys) + end + + content.map! do |lines| + next unless lines + + JSON.parse(lines).map! do |line| + line = line.with_indifferent_access + rich_text = line[:rich_text] + line[:rich_text] = rich_text&.html_safe + + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + def cache_key_for(raw_key) + "#{cache_key_prefix}:#{raw_key}" + end + + private + + def cache_key_prefix + "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight" + end + end + end + end +end diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb index 941244694e2..31bb6810391 100644 --- a/lib/gitlab/downtime_check.rb +++ b/lib/gitlab/downtime_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Checks if a set of migrations requires downtime or not. class DowntimeCheck diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 543e62794c5..ec38bd769a3 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class DowntimeCheck class Message @@ -18,13 +20,13 @@ module Gitlab def to_s label = offline ? OFFLINE : ONLINE - message = "[#{label}]: #{path}" + message = ["[#{label}]: #{path}"] if reason? - message += ":\n\n#{reason}\n\n" + message << ":\n\n#{reason}\n\n" end - message + message.join end def reason? diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 8c72d00c1f3..01fd261404b 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop: disable Rails/Output module Gitlab # Checks if a set of migrations requires downtime or not. @@ -5,7 +7,7 @@ module Gitlab CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb|locale/gitlab\.pot}i.freeze + IGNORED_FILES_REGEX = /VERSION|CHANGELOG\.md/i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== @@ -138,15 +140,23 @@ module Gitlab def ee_branch_presence_check! ee_remotes.keys.each do |remote| - [ce_branch, ee_branch_prefix, ee_branch_suffix].each do |branch| - _, status = step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + output, _ = step( + "Searching #{remote}", + %W[git ls-remote #{remote} *#{minimal_ee_branch_name}*]) - if status.zero? - @ee_remote_with_branch = remote - @ee_branch_found = branch - return true - end - end + branches = + output.scan(%r{(?<=refs/heads/|refs/tags/).+}).sort_by(&:size) + + next if branches.empty? + + branch = branches.first + + step("Fetching #{remote}/#{branch}", %W[git fetch #{remote} #{branch}]) + + @ee_remote_with_branch = remote + @ee_branch_found = branch + + return true end puts @@ -271,8 +281,12 @@ module Gitlab @ee_patch_full_path ||= patches_dir.join(ee_patch_name) end + def minimal_ee_branch_name + @minimal_ee_branch_name ||= ce_branch.sub(/(\Ace\-|\-ce\z)/, '') + end + def patch_name_from_branch(branch_name) - branch_name.parameterize << '.patch' + "#{branch_name.parameterize}.patch" end def patch_url @@ -420,9 +434,11 @@ module Gitlab end def conflicting_files_msg - failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file| - memo << "\n - #{file}" - end + header = "The conflicts detected were as follows:\n" + separator = "\n - " + failed_items = failed_files.join(separator) + + "#{header}#{separator}#{failed_items}" end end end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 83440ae227d..3323ce60158 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email class AttachmentUploader @@ -21,8 +23,8 @@ module Gitlab content_type: attachment.content_type } - link = UploadService.new(project, file).execute - attachments << link if link + uploader = UploadService.new(project, file).execute + attachments << uploader.to_h if uploader ensure tmp.close! end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index e08b5be8984..cebedb19dcc 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,20 +1,23 @@ -require 'gitlab/email/handler/create_merge_request_handler' -require 'gitlab/email/handler/create_note_handler' -require 'gitlab/email/handler/create_issue_handler' -require 'gitlab/email/handler/unsubscribe_handler' +# frozen_string_literal: true module Gitlab module Email module Handler - HANDLERS = [ - UnsubscribeHandler, - CreateNoteHandler, - CreateMergeRequestHandler, - CreateIssueHandler - ].freeze + def self.handlers + @handlers ||= load_handlers + end + + def self.load_handlers + [ + UnsubscribeHandler, + CreateNoteHandler, + CreateMergeRequestHandler, + CreateIssueHandler + ] + end def self.for(mail, mail_key) - HANDLERS.find do |klass| + handlers.find do |klass| handler = klass.new(mail, mail_key) break handler if handler.can_handle? end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 0bba433d04b..f89d1d15010 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -1,15 +1,19 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler class BaseHandler attr_reader :mail, :mail_key + HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze + def initialize(mail, mail_key) @mail = mail @mail_key = mail_key end - def can_execute? + def can_handle? raise NotImplementedError end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 764f93f6d3d..78a3a9489ac 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,20 +1,34 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' +# handles issue creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @incoming_email_token = - mail_key && mail_key.split('+', 2) + + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] + end end def can_handle? - !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) + incoming_email_token && (project_id || can_handle_legacy_format?) end def execute @@ -28,17 +42,11 @@ module Gitlab record_name: 'issue') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end - - def project - @project ||= Project.find_by_full_path(project_path) - end - - def metrics_params - super.merge(project: project&.full_path) - end + # rubocop: enable CodeReuse/ActiveRecord private @@ -50,6 +58,10 @@ module Gitlab description: message_including_reply ).execute end + + def can_handle_legacy_format? + project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY) + end end end end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index 2f864f2082b..b3b5063f2ca 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -1,23 +1,35 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles merge request creation emails with these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-merge-request@incoming.gitlab.com +# incoming+gitlab-org/gitlab-ce+merge-request+Author_Token12345678@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class CreateMergeRequestHandler < BaseHandler include ReplyProcessing - attr_reader :project_path, :incoming_email_token + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze def initialize(mail, mail_key) super(mail, mail_key) - if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) - @project_path, @incoming_email_token = m.captures + if !mail_key&.include?('/') && (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + elsif matched = HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @project_path = matched[:project_path] + @incoming_email_token = matched[:incoming_email_token] end end def can_handle? - @project_path && @incoming_email_token + incoming_email_token && (project_id || project_path) end def execute @@ -32,22 +44,32 @@ module Gitlab record_name: 'merge_request') end + # rubocop: disable CodeReuse/ActiveRecord def author @author ||= User.find_by(incoming_email_token: incoming_email_token) end - - def project - @project ||= Project.find_by_full_path(project_path) - end + # rubocop: enable CodeReuse/ActiveRecord def metrics_params - super.merge(project: project&.full_path) + super.merge(includes_patches: patch_attachments.any?) end private + def build_merge_request + MergeRequests::BuildService.new(project, author, merge_request_params).execute + end + def create_merge_request - merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + merge_request = build_merge_request + + if patch_attachments.any? + apply_patches_to_source_branch(start_branch: merge_request.target_branch) + remove_patch_attachments + # Rebuild the merge request as the source branch might just have + # been created, so we should re-validate. + merge_request = build_merge_request + end if merge_request.errors.any? merge_request @@ -59,12 +81,42 @@ module Gitlab def merge_request_params params = { source_project_id: project.id, - source_branch: mail.subject, + source_branch: source_branch, target_project_id: project.id } params[:description] = message if message.present? params end + + def apply_patches_to_source_branch(start_branch:) + patches = patch_attachments.map { |patch| patch.body.decoded } + + result = Commits::CommitPatchService + .new(project, author, branch_name: source_branch, patches: patches, start_branch: start_branch) + .execute + + if result[:status] != :success + message = "Could not apply patches to #{source_branch}:\n#{result[:message]}" + raise InvalidAttachment, message + end + end + + def remove_patch_attachments + patch_attachments.each { |patch| mail.parts.delete(patch) } + # reset the message, so it needs to be reprocessed when the attachments + # have been modified + @message = nil + end + + def patch_attachments + @patches ||= mail.attachments + .select { |attachment| attachment.filename.ends_with?('.patch') } + .sort_by(&:filename) + end + + def source_branch + @source_branch ||= mail.subject + end end end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 5791dbd0484..b00af15364d 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,6 +1,10 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' require 'gitlab/email/handler/reply_processing' +# handles note/reply creation emails with these formats: +# incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com module Gitlab module Email module Handler @@ -28,10 +32,6 @@ module Gitlab record_name: 'comment') end - def metrics_params - super.merge(project: project&.full_path) - end - private def author diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 38b1425364f..d8f4be8ada1 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -1,16 +1,31 @@ +# frozen_string_literal: true + module Gitlab module Email module Handler module ReplyProcessing private + attr_reader :project_id, :project_slug, :project_path, :incoming_email_token + def author raise NotImplementedError end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def project - raise NotImplementedError + return @project if instance_variable_defined?(:@project) + + if project_id + @project = Project.find_by_id(project_id) + @project = nil unless valid_project_slug?(@project) + else + @project = Project.find_by_full_path(project_path) + end + + @project end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def message @message ||= process_message @@ -41,7 +56,7 @@ module Gitlab raise ProjectNotFound unless author.can?(:read_project, project) end - raise UserNotAuthorizedError unless author.can?(permission, project || noteable) + raise UserNotAuthorizedError unless author.can?(permission, try(:noteable) || project) end def verify_record!(record:, invalid_exception:, record_name:) @@ -56,6 +71,10 @@ module Gitlab raise invalid_exception, msg end + + def valid_project_slug?(found_project) + project_slug == found_project.full_path_slug + end end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index ea80e21532e..20e4c125626 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -1,13 +1,29 @@ +# frozen_string_literal: true + require 'gitlab/email/handler/base_handler' +# handles unsubscribe emails with these formats: +# incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com +# incoming+1234567890abcdef1234567890abcdef+unsubscribe@incoming.gitlab.com (legacy) module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler delegate :project, to: :sent_notification, allow_nil: true + HANDLER_REGEX_FOR = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze + HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze + HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze + + def initialize(mail, mail_key) + super(mail, mail_key) + + matched = HANDLER_REGEX.match(mail_key.to_s) || HANDLER_REGEX_LEGACY.match(mail_key.to_s) + @reply_token = matched[:reply_token] if matched + end + def can_handle? - mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + reply_token.present? end def execute @@ -20,18 +36,12 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end - def metrics_params - super.merge(project: project&.full_path) - end - private - def sent_notification - @sent_notification ||= SentNotification.for(reply_key) - end + attr_reader :reply_token - def reply_key - mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + def sent_notification + @sent_notification ||= SentNotification.for(reply_token) end end end diff --git a/lib/gitlab/email/hook/additional_headers_interceptor.rb b/lib/gitlab/email/hook/additional_headers_interceptor.rb new file mode 100644 index 00000000000..aa2ef76069b --- /dev/null +++ b/lib/gitlab/email/hook/additional_headers_interceptor.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class AdditionalHeadersInterceptor + def self.delivering_email(message) + message.header['Auto-Submitted'] ||= 'auto-generated' + message.header['X-Auto-Response-Suppress'] ||= 'All' + end + end + end + end +end diff --git a/lib/gitlab/email/hook/delivery_metrics_observer.rb b/lib/gitlab/email/hook/delivery_metrics_observer.rb new file mode 100644 index 00000000000..c7af485fcc5 --- /dev/null +++ b/lib/gitlab/email/hook/delivery_metrics_observer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class DeliveryMetricsObserver + extend Gitlab::Utils::StrongMemoize + + def self.delivering_email(_message) + delivery_attempts_counter.increment + end + + def self.delivered_email(_message) + delivered_emails_counter.increment + end + + def self.delivery_attempts_counter + strong_memoize(:delivery_attempts_counter) do + Gitlab::Metrics.counter(:gitlab_emails_delivery_attempts_total, + 'Counter of total emails delivery attempts') + end + end + + def self.delivered_emails_counter + strong_memoize(:delivered_emails_counter) do + Gitlab::Metrics.counter(:gitlab_emails_delivered_total, + 'Counter of total emails delievered') + end + end + end + end + end +end diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb new file mode 100644 index 00000000000..6b6b1d85109 --- /dev/null +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class DisableEmailInterceptor + def self.delivering_email(message) + message.perform_deliveries = false + + Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" + end + end + end + end +end diff --git a/lib/gitlab/email/hook/email_template_interceptor.rb b/lib/gitlab/email/hook/email_template_interceptor.rb new file mode 100644 index 00000000000..13f8db2051d --- /dev/null +++ b/lib/gitlab/email/hook/email_template_interceptor.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class EmailTemplateInterceptor + ## + # Remove HTML part if HTML emails are disabled. + # + def self.delivering_email(message) + unless Gitlab::CurrentSettings.html_emails_enabled + message.parts.delete_if do |part| + part.content_type.start_with?('text/html') + end + end + end + end + end + end +end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 50559a48973..77f299bcade 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email class HTMLParser diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index cd9d3a6483f..ec412e7a8b1 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Message @@ -116,7 +118,7 @@ module Gitlab end def subject - subject_text = '[Git]' + subject_text = ['[Git]'] subject_text << "[#{project.full_path}]" subject_text << "[#{ref_name}]" if @action == :push subject_text << ' ' @@ -134,6 +136,8 @@ module Gitlab subject_action[0] = subject_action[0].capitalize subject_text << "#{subject_action} #{ref_type} #{ref_name}" end + + subject_text.join end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d8c594ad0e7..d28f6b301fa 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -18,6 +20,7 @@ module Gitlab InvalidIssueError = Class.new(InvalidRecordError) InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) + InvalidAttachment = Class.new(ProcessingError) class Receiver def initialize(raw) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index ae6b84607d6..2743f011ca6 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Inspired in great part by Discourse's Email::Receiver module Gitlab module Email diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 89cf659bce4..ce1dfb0753c 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Emoji extend self diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 0b8f6cfe3cb..a4a154c80f7 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EncodingHelper extend self @@ -65,17 +67,17 @@ module Gitlab clean(message) end rescue ArgumentError - return nil + nil end - def encode_binary(s) - return "" if s.nil? + def encode_binary(str) + return "" if str.nil? - s.dup.force_encoding(Encoding::ASCII_8BIT) + str.dup.force_encoding(Encoding::ASCII_8BIT) end - def binary_stringio(s) - StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } + def binary_stringio(str) + StringIO.new(str.freeze || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index 5e0dd6e7859..b1a9603d3a5 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Environment def self.hostname diff --git a/lib/gitlab/environment_logger.rb b/lib/gitlab/environment_logger.rb index 407cc572656..862a516ca71 100644 --- a/lib/gitlab/environment_logger.rb +++ b/lib/gitlab/environment_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class EnvironmentLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/error_tracking/error.rb b/lib/gitlab/error_tracking/error.rb new file mode 100644 index 00000000000..4af5192aa6a --- /dev/null +++ b/lib/gitlab/error_tracking/error.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Error + include ActiveModel::Model + + attr_accessor :id, :title, :type, :user_count, :count, + :first_seen, :last_seen, :message, :culprit, + :external_url, :project_id, :project_name, :project_slug, + :short_id, :status, :frequency + end + end +end diff --git a/lib/gitlab/error_tracking/project.rb b/lib/gitlab/error_tracking/project.rb new file mode 100644 index 00000000000..93e81da5034 --- /dev/null +++ b/lib/gitlab/error_tracking/project.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class Project + include ActiveModel::Model + + ACCESSORS = [ + :id, :name, :status, :slug, :organization_name, + :organization_id, :organization_slug + ].freeze + + attr_accessor(*ACCESSORS) + end + end +end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index d5d35dbd97f..a11d6b66409 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Middleware @@ -6,7 +8,7 @@ module Gitlab end def call(env) - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) route = Gitlab::EtagCaching::Router.match(request.path_info) return @app.call(env) unless route diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 75167a6b088..08e30214b46 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Router diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 21172ff8d93..2395e7be026 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Store diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 12b5e240962..d466d2a514c 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'securerandom' module Gitlab diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb new file mode 100644 index 00000000000..7961d4bbd6e --- /dev/null +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + # This module provides helper methods which are intregrated with GitLab::ExclusiveLease + module ExclusiveLeaseHelpers + FailedToObtainLockError = Class.new(StandardError) + + ## + # This helper method blocks a process/thread until the other process cancel the obrainted lease key. + # + # Note: It's basically discouraged to use this method in the unicorn's thread, + # because it holds the connection until all `retries` is consumed. + # This could potentially eat up all connection pools. + def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds) + raise ArgumentError, 'Key needs to be specified' unless key + + lease = Gitlab::ExclusiveLease.new(key, timeout: ttl) + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit. + sleep(sleep_sec) + break if (retries -= 1) < 0 + end + + raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid + + yield + ensure + Gitlab::ExclusiveLease.cancel(key, uuid) + end + end +end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bb14a8cd9e7..bd806269bf0 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This class extends an OpenStruct object by adding predicate methods to mimic # ActiveRecord access. We rely on the initial values being true or false to # determine whether to define a predicate method because for a newly-added @@ -5,12 +7,6 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct - def initialize(options = {}) - super - - FakeApplicationSettings.define_predicate_methods(options) - end - # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -23,5 +19,27 @@ module Gitlab end end end + + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + def key_restriction_for(type) + 0 + end + + def allowed_key_types + ApplicationSetting::SUPPORTED_KEY_TYPES + end + + def pick_repository_storage + repository_storages.sample + end + + def commit_email_hostname + super.presence || ApplicationSetting.default_commit_email_hostname + end end end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb new file mode 100644 index 00000000000..1ae2f9dfd93 --- /dev/null +++ b/lib/gitlab/favicon.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + class Favicon + class << self + def main + image_name = + if appearance_favicon.exists? + appearance_favicon.url + elsif Gitlab::Utils.to_boolean(ENV['CANARY']) + 'favicon-yellow.png' + elsif Rails.env.development? + 'favicon-blue.png' + else + 'favicon.png' + end + + ActionController::Base.helpers.image_path(image_name, host: host) + end + + def status_overlay(status_name) + path = File.join( + 'ci_favicons', + "#{status_name}.png" + ) + + ActionController::Base.helpers.image_path(path, host: host) + end + + def available_status_names + @available_status_names ||= begin + Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) + .map { |file| File.basename(file, '.png') } + .sort + end + end + + private + + # we only want to create full urls when there's a different asset_host + # configured. + def host + asset_host = ActionController::Base.asset_host + if asset_host.nil? || asset_host == Gitlab.config.gitlab.base_url + nil + else + Gitlab.config.gitlab.base_url + end + end + + def appearance + Gitlab::SafeRequestStore[:appearance] ||= (Appearance.current || Appearance.new) + end + + def appearance_favicon + appearance.favicon + end + end + end +end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 49bc9c0b671..2770469ca9f 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'set' module Gitlab @@ -6,9 +8,9 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: %r{\Areadme[^/]*\z}i, + readme: /\A(#{Regexp.union(*Gitlab::MarkupHelper::PLAIN_FILENAMES).source})(\.(#{Regexp.union(*Gitlab::MarkupHelper::EXTENSIONS).source}))?\z/i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, - license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, @@ -18,7 +20,6 @@ module Gitlab # Configuration files gitignore: '.gitignore', - koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', route_map: '.gitlab/route-map.yml', diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 8c082c0c336..3958814208c 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -1,9 +1,9 @@ +# frozen_string_literal: true + # This class finds files in a repository by name and content # the result is joined and sorted by file name module Gitlab class FileFinder - BATCH_SIZE = 100 - attr_reader :project, :ref delegate :repository, to: :project @@ -14,41 +14,35 @@ module Gitlab end def find(query) - by_content = find_by_content(query) + query = Gitlab::Search::Query.new(query, encode_binary: true) do + filter :filename, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.binary_filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.binary_filename =~ /\.#{filter[:regex_value]}$/i } + end + + files = find_by_filename(query.term) + find_by_content(query.term) - already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query, except: already_found) + files = query.filter_results(files) if query.filters.any? - (by_content + by_filename) - .sort_by(&:filename) - .map { |blob| [blob.filename, blob] } + files end private def find_by_content(query) - results = repository.search_files_by_content(query, ref).first(BATCH_SIZE) - results.map { |result| Gitlab::ProjectSearchResults.parse_search_result(result, project) } + repository.search_files_by_content(query, ref).map do |result| + Gitlab::Search::FoundBlob.new(content_match: result, project: project, ref: ref, repository: repository) + end end - def find_by_filename(query, except: []) - filenames = repository.search_files_by_name(query, ref).first(BATCH_SIZE) - filenames.delete_if { |filename| except.include?(filename) } unless except.empty? - - blob_refs = filenames.map { |filename| [ref, filename] } - blobs = Gitlab::Git::Blob.batch(repository, blob_refs, blob_size_limit: 1024) - - blobs.map do |blob| - Gitlab::SearchResults::FoundBlob.new( - id: blob.id, - filename: blob.path, - basename: File.basename(blob.path), - ref: ref, - startline: 1, - data: blob.data, - project: project - ) + def find_by_filename(query) + search_filenames(query).map do |filename| + Gitlab::Search::FoundBlob.new(blob_filename: filename, project: project, ref: ref, repository: repository) end end + + def search_filenames(query) + repository.search_files_by_name(query, ref) + end end end diff --git a/lib/gitlab/file_markdown_link_builder.rb b/lib/gitlab/file_markdown_link_builder.rb new file mode 100644 index 00000000000..180140e7da2 --- /dev/null +++ b/lib/gitlab/file_markdown_link_builder.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# Builds the markdown link of a file +# It needs the methods filename and secure_url (final destination url) to be defined. +module Gitlab + module FileMarkdownLinkBuilder + include FileTypeDetection + + def markdown_link + return unless name = markdown_name + + markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})" + markdown = "!#{markdown}" if image_or_video? || dangerous? + markdown + end + + def markdown_name + return unless filename.present? + + image_or_video? ? File.basename(filename, File.extname(filename)) : filename + end + end +end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb new file mode 100644 index 00000000000..25ee07cf940 --- /dev/null +++ b/lib/gitlab/file_type_detection.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# File helpers methods. +# It needs the method filename to be defined. +module Gitlab + module FileTypeDetection + IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + # We recommend using the .mp4 format over .mov. Videos in .mov format can + # still be used but you really need to make sure they are served with the + # proper MIME type video/mp4 and not video/quicktime or your videos won't play + # on IE >= 9. + # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html + VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze + # These extension types can contain dangerous code and should only be embedded inline with + # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline". + DANGEROUS_EXT = %w[svg].freeze + + def image? + extension_match?(IMAGE_EXT) + end + + def video? + extension_match?(VIDEO_EXT) + end + + def image_or_video? + image? || video? + end + + def dangerous? + extension_match?(DANGEROUS_EXT) + end + + private + + def extension_match?(extensions) + return false unless filename + + extension = File.extname(filename).delete('.') + extensions.include?(extension.downcase) + end + end +end diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index acb000e3e23..dd747a79673 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'fogbugz' module Gitlab diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 8953bc8c148..431911d1eee 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module FogbugzImport class Importer @@ -79,6 +81,7 @@ module Gitlab ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end + # rubocop: disable CodeReuse/ActiveRecord def user_info(person_id) user_hash = user_map[person_id.to_s] @@ -95,7 +98,9 @@ module Gitlab { name: user_name, gitlab_id: gitlab_id } end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def import_cases return unless @cases @@ -141,6 +146,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def opened_content(comments) while comment = comments.shift @@ -191,19 +197,19 @@ module Gitlab end end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') + str end - def escape_for_markdown(s) - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") - s = s.gsub("`", "\\~") - s = s.delete("\r") - s = s.gsub("\n", " \n") - s + def escape_for_markdown(str) + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") + str = str.gsub("`", "\\~") + str = str.delete("\r") + str = str.gsub("\n", " \n") + str end def format_content(raw_content) diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 1918d5b208d..3c71031a8d9 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module FogbugzImport class ProjectCreator diff --git a/lib/gitlab/fogbugz_import/repository.rb b/lib/gitlab/fogbugz_import/repository.rb index d1dc63db2b2..b958dcf6cbf 100644 --- a/lib/gitlab/fogbugz_import/repository.rb +++ b/lib/gitlab/fogbugz_import/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module FogbugzImport class Repository diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 455814a9159..4d82acd9d87 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Gfm ## @@ -31,19 +33,19 @@ module Gitlab class ReferenceRewriter RewriteError = Class.new(StandardError) - def initialize(text, source_project, current_user) + def initialize(text, source_parent, current_user) @text = text - @source_project = source_project + @source_parent = source_parent @current_user = current_user @original_html = markdown(text) @pattern = Gitlab::ReferenceExtractor.references_pattern end - def rewrite(target_project) + def rewrite(target_parent) return @text unless needs_rewrite? @text.gsub(@pattern) do |reference| - unfold_reference(reference, Regexp.last_match, target_project) + unfold_reference(reference, Regexp.last_match, target_parent) end end @@ -53,14 +55,14 @@ module Gitlab private - def unfold_reference(reference, match, target_project) + def unfold_reference(reference, match, target_parent) before = @text[0...match.begin(0)] after = @text[match.end(0)..-1] referable = find_referable(reference) return reference unless referable - cross_reference = build_cross_reference(referable, target_project) + cross_reference = build_cross_reference(referable, target_parent) return reference if reference == cross_reference if cross_reference.nil? @@ -72,17 +74,17 @@ module Gitlab end def find_referable(reference) - extractor = Gitlab::ReferenceExtractor.new(@source_project, + extractor = Gitlab::ReferenceExtractor.new(@source_parent, @current_user) extractor.analyze(reference) extractor.all.first end - def build_cross_reference(referable, target_project) + def build_cross_reference(referable, target_parent) if referable.respond_to?(:project) - referable.to_reference(target_project) + referable.to_reference(target_parent) else - referable.to_reference(@source_project, target_project: target_project) + referable.to_reference(@source_parent, target_project: target_parent) end end @@ -91,7 +93,7 @@ module Gitlab end def markdown(text) - Banzai.render(text, project: @source_project, no_original_data: true) + Banzai.render(text, project: @source_parent, no_original_data: true, no_sourcepos: true) end end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index b6eeb5d9a2b..2d1c9ac40ae 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'fileutils' module Gitlab @@ -6,7 +8,7 @@ module Gitlab # Class that rewrites markdown links for uploads # # Using a pattern defined in `FileUploader` it copies files to a new - # project and rewrites all links to uploads in in a given text. + # project and rewrites all links to uploads in a given text. # # class UploadsRewriter @@ -16,18 +18,16 @@ module Gitlab @pattern = FileUploader::MARKDOWN_PATTERN end - def rewrite(target_project) + def rewrite(target_parent) return @text unless needs_rewrite? @text.gsub(@pattern) do |markdown| file = find_file(@source_project, $~[:secret], $~[:file]) break markdown unless file.try(:exists?) - new_uploader = FileUploader.new(target_project) - with_link_in_tmp_dir(file.file) do |open_tmp_file| - new_uploader.store!(open_tmp_file) - end - new_uploader.markdown_link + klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader + moved = klass.copy_to(file, target_parent) + moved.markdown_link end end @@ -48,20 +48,7 @@ module Gitlab def find_file(project, secret, file) uploader = FileUploader.new(project, secret: secret) uploader.retrieve_from_store!(file) - uploader.file - end - - # Because the uploaders use 'move_to_store' we must have a temporary - # file that is allowed to be (re)moved. - def with_link_in_tmp_dir(file) - dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) - # The filename matters to Carrierwave so we make sure to preserve it - tmp_file = File.join(dir, File.basename(file)) - File.link(file, tmp_file) - # Open the file to placate Carrierwave - File.open(tmp_file) { |open_file| yield open_file } - ensure - FileUtils.rm_rf(dir) + uploader end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index e85e87a54af..44a62586a23 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'gitlab/encoding_helper' module Gitlab @@ -10,9 +12,11 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze - CommandError = Class.new(StandardError) - CommitError = Class.new(StandardError) - OSError = Class.new(StandardError) + BaseError = Class.new(StandardError) + CommandError = Class.new(BaseError) + CommitError = Class.new(BaseError) + OSError = Class.new(BaseError) + UnknownRef = Class.new(BaseError) class << self include Gitlab::EncodingHelper @@ -50,11 +54,11 @@ module Gitlab end def tag_ref?(ref) - ref.start_with?(TAG_REF_PREFIX) + ref =~ /^#{TAG_REF_PREFIX}.+/ end def branch_ref?(ref) - ref.start_with?(BRANCH_REF_PREFIX) + ref =~ /^#{BRANCH_REF_PREFIX}.+/ end def blank_ref?(ref) @@ -62,7 +66,7 @@ module Gitlab end def version - Gitlab::VersionInfo.parse(Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --version)).first) + Gitlab::Git::Version.git_version end def check_namespace!(*objects) diff --git a/lib/gitlab/git/attributes_at_ref_parser.rb b/lib/gitlab/git/attributes_at_ref_parser.rb index 26b5bd520d5..cbddf836ce8 100644 --- a/lib/gitlab/git/attributes_at_ref_parser.rb +++ b/lib/gitlab/git/attributes_at_ref_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git # Parses root .gitattributes file at a given ref diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb index 08f4d7d4f5c..8b9d74ae8e7 100644 --- a/lib/gitlab/git/attributes_parser.rb +++ b/lib/gitlab/git/attributes_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git # Class for parsing Git attribute files and extracting the attributes for diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 4158d50cd9e..b118eda37f8 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git class Blame @@ -22,24 +24,9 @@ module Gitlab private def load_blame - raw_output = @repo.gitaly_migrate(:blame) do |is_enabled| - if is_enabled - load_blame_by_gitaly - else - load_blame_by_shelling_out - end - end - - output = encode_utf8(raw_output) - process_raw_blame output - end - - def load_blame_by_gitaly - @repo.gitaly_commit_client.raw_blame(@sha, @path) - end + output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path)) - def load_blame_by_shelling_out - @repo.shell_blame(@sha, @path) + process_raw_blame(output) end def process_raw_blame(output) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 156d077a69c..259a2b7911a 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -1,15 +1,16 @@ -# Gitaly note: JV: seems to be completely migrated (behind feature flags). +# frozen_string_literal: true module Gitlab module Git class Blob - include Linguist::BlobHelper + include Gitlab::BlobHelper include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors # This number is the maximum amount of data that we want to display to - # the user. We load as much as we can for encoding detection - # (Linguist) and LFS pointer parsing. All other cases where we need full - # blob data should use load_all_data!. + # the user. We load as much as we can for encoding detection and LFS + # pointer parsing. All other cases where we need full blob data should + # use load_all_data!. MAX_DATA_DISPLAY_SIZE = 10.megabytes # These limits are used as a heuristic to ignore files which can't be LFS @@ -21,24 +22,36 @@ module Gitlab attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self - def find(repository, sha, path) - Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| - if is_enabled - find_by_gitaly(repository, sha, path) - else - find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - end + def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') + path = '/' if path.empty? + name = File.basename(path) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) + return unless entry + + entry.data = "" if limit == 0 + + case entry.type + when :COMMIT + new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha) + when :BLOB + new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8), + path: path, commit_id: sha, binary: binary?(entry.data)) end end def raw(repository, sha) - Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) - else - rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) - end - end + repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) end # Returns an array of Blob instances, specified in blob_references as @@ -49,17 +62,8 @@ module Gitlab # Keep in mind that this method may allocate a lot of memory. It is up # to the caller to limit the number of blobs and blob_size_limit. # - # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) - Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a - else - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) - end - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a end # Returns an array of Blob instances just with the metadata, that means @@ -72,16 +76,8 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled| - if is_enabled - repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) - else - blob_ids.lazy - .select { |sha| possible_lfs_blob?(repository, sha) } - .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) } - .select(&:lfs_pointer?) - .force - end + wrapped_gitaly_errors do + repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) end end @@ -92,151 +88,6 @@ module Gitlab def size_could_be_lfs?(size) size.between?(LFS_POINTER_MIN_SIZE, LFS_POINTER_MAX_SIZE) end - - private - - # Recursive search of blob id by path - # - # Ex. - # blog/ # oid: 1a - # app/ # oid: 2a - # models/ # oid: 3a - # file.rb # oid: 4a - # - # - # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' - # - def find_entry_by_path(repository, root_id, *path_parts) - root_tree = repository.lookup(root_id) - - entry = root_tree.find do |entry| - entry[:name] == path_parts[0] - end - - return nil unless entry - - if path_parts.size > 1 - return nil unless entry[:type] == :tree - - path_parts.shift - find_entry_by_path(repository, entry[:oid], *path_parts) - else - [:blob, :commit].include?(entry[:type]) ? entry : nil - end - end - - def submodule_blob(blob_entry, path, sha) - new( - id: blob_entry[:oid], - name: blob_entry[:name], - size: 0, - data: '', - path: path, - commit_id: sha - ) - end - - def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - return unless path - - path = path.sub(%r{\A/*}, '') - path = '/' if path.empty? - name = File.basename(path) - - # Gitaly will think that setting the limit to 0 means unlimited, while - # the client might only need the metadata and thus set the limit to 0. - # In this method we'll then set the limit to 1, but clear the byte of data - # that we got back so for the outside world it looks like the limit was - # actually 0. - req_limit = limit == 0 ? 1 : limit - - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) - return unless entry - - entry.data = "" if limit == 0 - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path, limit:) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - - def rugged_raw(repository, sha, limit:) - blob = repository.lookup(sha) - - return unless blob.is_a?(Rugged::Blob) - - new( - id: blob.oid, - size: blob.size, - data: blob.content(limit), - binary: blob.binary? - ) - end - - # Efficient lookup to determine if object size - # and type make it a possible LFS blob without loading - # blob content into memory with repository.lookup(sha) - def possible_lfs_blob?(repository, sha) - object_header = repository.rugged.read_header(sha) - - object_header[:type] == :blob && - size_could_be_lfs?(object_header[:len]) - end end def initialize(options) @@ -249,7 +100,7 @@ module Gitlab @loaded_all_data = @loaded_size == size end - def binary? + def binary_in_repo? @binary.nil? ? super : @binary == true end @@ -268,16 +119,7 @@ module Gitlab return if @loaded_all_data - @data = Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled| - begin - if is_enabled - repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data - else - repository.lookup(id).content - end - end - end - + @data = repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data @loaded_all_data = true @loaded_size = @data.bytesize end @@ -332,7 +174,7 @@ module Gitlab private def has_lfs_version_key? - !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec") end end end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb deleted file mode 100644 index 68116e775c6..00000000000 --- a/lib/gitlab/git/blob_snippet.rb +++ /dev/null @@ -1,34 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -module Gitlab - module Git - class BlobSnippet - include Linguist::BlobHelper - - attr_accessor :ref - attr_accessor :lines - attr_accessor :filename - attr_accessor :startline - - def initialize(ref, lines, startline, filename) - @ref, @lines, @startline, @filename = ref, lines, startline, filename - end - - def data - lines&.join("\n") - end - - def name - filename - end - - def size - data.length - end - - def mode - nil - end - end - end -end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 6351cfb83e3..9447cfa0fb6 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git class Branch < Ref diff --git a/lib/gitlab/git/bundle_file.rb b/lib/gitlab/git/bundle_file.rb new file mode 100644 index 00000000000..8384a436fcc --- /dev/null +++ b/lib/gitlab/git/bundle_file.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class BundleFile + # All git bundle files start with this string + # + # https://github.com/git/git/blob/v2.20.1/bundle.c#L15 + MAGIC = "# v2 git bundle\n" + + InvalidBundleError = Class.new(StandardError) + + attr_reader :filename + + def self.check!(filename) + new(filename).check! + end + + def initialize(filename) + @filename = filename + end + + def check! + data = File.open(filename, 'r') { |f| f.read(MAGIC.size) } + + raise InvalidBundleError, 'Invalid bundle file' unless data == MAGIC + end + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 89f761dd515..5863815ca85 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -1,8 +1,11 @@ -# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object +# frozen_string_literal: true + +# Gitlab::Git::Commit is a wrapper around Gitaly::GitCommit module Gitlab module Git class Commit include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -53,34 +56,21 @@ module Gitlab # Already a commit? return commit_id if commit_id.is_a?(Gitlab::Git::Commit) - # A rugged reference? - commit_id = Gitlab::Git::Ref.dereference_object(commit_id) - return decorate(repo, commit_id) if commit_id.is_a?(Rugged::Commit) - # Some weird thing? return nil unless commit_id.is_a?(String) - commit = repo.gitaly_migrate(:find_commit) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.find_commit(commit_id) - else - rugged_find(repo, commit_id) - end + # This saves us an RPC round trip. + return nil if commit_id.include?(':') + + commit = wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) end decorate(repo, commit) if commit - rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, - Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, - Rugged::OdbError, Rugged::TreeError, ArgumentError + rescue Gitlab::Git::CommandError, Gitlab::Git::Repository::NoRepository, ArgumentError nil end - def rugged_find(repo, commit_id) - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil - end - # Get last commit for HEAD # # Ex. @@ -113,15 +103,9 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.between(base, head) - else - repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } - end + wrapped_gitaly_errors do + repo.gitaly_commit_client.between(base, head) end - rescue Rugged::ReferenceError - [] end # Returns commits collection @@ -143,190 +127,55 @@ module Gitlab # :topo, or any combination of them (in an array). Commit ordering types # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| - if is_enabled - find_all_by_gitaly(repo, options) - else - find_all_by_rugged(repo, options) - end - end - end - - def find_all_by_rugged(repo, options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) - end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - rugged = repo.rugged - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end - end - - walker.sorting(rugged_sort_type(actual_options[:order])) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(repo, commit)) + wrapped_gitaly_errors do + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - - def find_all_by_gitaly(repo, options = {}) - Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(repository, commit, ref = nil) Gitlab::Git::Commit.new(repository, commit, ref) end - # Returns the `Rugged` sorting type constant for one or more given - # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array - # containing more than one of them. `:date` uses a combination of date and - # topological sorting to closer mimic git's native ordering. - def rugged_sort_type(sort_type) - @rugged_sort_types ||= { - none: Rugged::SORT_NONE, - topo: Rugged::SORT_TOPO, - date: Rugged::SORT_DATE | Rugged::SORT_TOPO - } - - @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) - end - def shas_with_signatures(repository, shas) - GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) - else - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false - end - end - end - end + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) end # Only to be used when the object ids will not necessarily have a # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.gitaly_migrate(:list_commits_by_oid, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.list_commits_by_oid(oids) - else - oids.map { |oid| find(repo, oid) }.compact - end + wrapped_gitaly_errors do + repo.gitaly_commit_client.list_commits_by_oid(oids) end end def extract_signature(repository, commit_id) - repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.extract_signature(commit_id) - else - rugged_extract_signature(repository, commit_id) - end - end + repository.gitaly_commit_client.extract_signature(commit_id) end def extract_signature_lazily(repository, commit_id) - BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| - items_by_repo = items.group_by { |i| i[:repository] } - - items_by_repo.each do |repo, items| - commit_ids = items.map { |i| i[:commit_id] } - - signatures = batch_signature_extraction(repository, commit_ids) - - signatures.each do |commit_sha, signature_data| - loader.call({ repository: repository, commit_id: commit_sha }, signature_data) - end + BatchLoader.for(commit_id).batch(key: repository) do |commit_ids, loader, args| + batch_signature_extraction(args[:key], commit_ids).each do |commit_id, signature_data| + loader.call(commit_id, signature_data) end end end def batch_signature_extraction(repository, commit_ids) - repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| - if is_enabled - gitaly_batch_signature_extraction(repository, commit_ids) - else - rugged_batch_signature_extraction(repository, commit_ids) - end - end - end - - def gitaly_batch_signature_extraction(repository, commit_ids) repository.gitaly_commit_client.get_commit_signatures(commit_ids) end - def rugged_batch_signature_extraction(repository, commit_ids) - commit_ids.each_with_object({}) do |commit_id, signatures| - signature_data = rugged_extract_signature(repository, commit_id) - next unless signature_data - - signatures[commit_id] = signature_data - end - end - - def rugged_extract_signature(repository, commit_id) - begin - Rugged::Commit.extract_signature(repository.rugged, commit_id) - rescue Rugged::OdbError - nil - end - end - def get_message(repository, commit_id) - BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| - items_by_repo = items.group_by { |i| i[:repository] } - - items_by_repo.each do |repo, items| - commit_ids = items.map { |i| i[:commit_id] } - - messages = get_messages(repository, commit_ids) - - messages.each do |commit_sha, message| - loader.call({ repository: repository, commit_id: commit_sha }, message) - end + BatchLoader.for(commit_id).batch(key: repository) do |commit_ids, loader, args| + get_messages(args[:key], commit_ids).each do |commit_id, message| + loader.call(commit_id, message) end end end def get_messages(repository, commit_ids) - repository.gitaly_migrate(:commit_messages) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.get_commit_messages(commit_ids) - else - commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h - end - end + repository.gitaly_commit_client.get_commit_messages(commit_ids) end end @@ -339,8 +188,6 @@ module Gitlab case raw_commit when Hash init_from_hash(raw_commit) - when Rugged::Commit - init_from_rugged(raw_commit) when Gitaly::GitCommit init_from_gitaly(raw_commit) else @@ -378,40 +225,12 @@ module Gitlab # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end - end - - def rugged_diff_from_parent(options = {}) - options ||= {} - break_rewrites = options[:break_rewrites] - actual_options = Gitlab::Git::Diff.filter_diff_options(options) - - diff = if rugged_commit.parents.empty? - rugged_commit.diff(actual_options.merge(reverse: true)) - else - rugged_commit.parents[0].diff(rugged_commit, actual_options) - end - - diff.find_similar!(break_rewrites: break_rewrites) - diff + @repository.gitaly_commit_client.diff_from_parent(self, options) end def deltas @deltas ||= begin - deltas = Gitlab::GitalyClient.migrate(:commit_deltas) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.commit_deltas(self) - else - rugged_diff_from_parent.each_delta - end - end - + deltas = @repository.gitaly_commit_client.commit_deltas(self) deltas.map { |delta| Gitlab::Git::Diff.new(delta) } end end @@ -479,14 +298,6 @@ module Gitlab encode! @committer_email end - def rugged_commit - @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) - raw_commit - else - @repository.rev_parse_target(id) - end - end - def merge_commit? parent_ids.size > 1 end @@ -494,13 +305,17 @@ module Gitlab def tree_entry(path) return unless path.present? - @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| - if is_migrated - gitaly_tree_entry(path) - else - rugged_tree_entry(path) - end - end + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry end def to_gitaly_commit @@ -512,8 +327,8 @@ module Gitlab subject: message_split[0] ? message_split[0].chomp.b : "", body: raw_commit.message.b, parent_ids: raw_commit.parent_ids, - author: gitaly_commit_author_from_rugged(raw_commit.author), - committer: gitaly_commit_author_from_rugged(raw_commit.committer) + author: gitaly_commit_author_from_raw(raw_commit.author), + committer: gitaly_commit_author_from_raw(raw_commit.committer) ) end @@ -527,22 +342,6 @@ module Gitlab end end - def init_from_rugged(commit) - author = commit.author - committer = commit.committer - - @raw_commit = commit - @id = commit.oid - @message = commit.message - @authored_date = author[:time] - @committed_date = committer[:time] - @author_name = author[:name] - @author_email = author[:email] - @committer_name = committer[:name] - @committer_email = committer[:email] - @parent_ids = commit.parents.map(&:oid) - end - def init_from_gitaly(commit) @raw_commit = commit @id = commit.id @@ -563,29 +362,7 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_tree_entry(path) - # We're only interested in metadata, so limit actual data to 1 byte - # since Gitaly doesn't support "send no data" option. - entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) - return unless entry - - # To be compatible with the rugged format - entry = entry.to_h - entry.delete(:data) - entry[:name] = File.basename(path) - entry[:type] = entry[:type].downcase - - entry - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - - def gitaly_commit_author_from_rugged(author_or_committer) + def gitaly_commit_author_from_raw(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, email: author_or_committer[:email].b, diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 8463b1eb794..8815088d23c 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,10 +1,12 @@ -# Gitaly note: JV: 1 RPC, migration in progress. +# frozen_string_literal: true # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab module Git class CommitStats + include Gitlab::Git::WrapsGitalyErrors + attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object @@ -16,12 +18,8 @@ module Gitlab @deletions = 0 @total = 0 - repo.gitaly_migrate(:commit_stats) do |is_enabled| - if is_enabled - gitaly_stats(repo, commit) - else - rugged_stats(commit) - end + wrapped_gitaly_errors do + gitaly_stats(repo, commit) end end @@ -31,12 +29,6 @@ module Gitlab @deletions = stats.deletions @total = @additions + @deletions end - - def rugged_stats(commit) - diff = commit.rugged_diff_from_parent - _files_changed, @additions, @deletions = diff.stat - @total = @additions + @deletions - end end end end diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb deleted file mode 100644 index a8a59f998cd..00000000000 --- a/lib/gitlab/git/committer_with_hooks.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Git - class CommitterWithHooks < Gollum::Committer - attr_reader :gl_wiki - - def initialize(gl_wiki, options = {}) - @gl_wiki = gl_wiki - super(gl_wiki.gollum_wiki, options) - end - - def commit - # TODO: Remove after 10.8 - return super unless allowed_to_run_hooks? - - result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( - @wiki.ref, - start_branch_name: @wiki.ref - ) do |start_commit| - super(false) - end - - result[:newrev] - rescue Gitlab::Git::HooksService::PreReceiveError => e - message = "Custom Hook failed: #{e.message}" - raise Gitlab::Git::Wiki::OperationError, message - end - - private - - # TODO: Remove after 10.8 - def allowed_to_run_hooks? - @options[:user_id] != 0 && @options[:username].present? - end - - def git_user - @git_user ||= Gitlab::Git::User.new(@options[:username], - @options[:name], - @options[:email], - gitlab_id) - end - - def gitlab_id - Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) - end - end - end -end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 7cb842256d0..ab5245ba7cb 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index f08dab59ce4..7ffe4a7ae81 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git module Conflict diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index fb5717dd556..20de8ebde4e 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git module Conflict diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb index ab9be683e15..04299a2d10c 100644 --- a/lib/gitlab/git/conflict/resolution.rb +++ b/lib/gitlab/git/conflict/resolution.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git module Conflict diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index c3cb0264112..26e82643a4c 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Git module Conflict class Resolver + include Gitlab::Git::WrapsGitalyErrors + ConflictSideMissing = Class.new(StandardError) ResolutionError = Class.new(StandardError) @@ -12,28 +16,18 @@ module Gitlab end def conflicts - @conflicts ||= begin - @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| - if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files.to_a - else - rugged_list_conflict_files - end - end + @conflicts ||= wrapped_gitaly_errors do + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) - rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e + rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) - source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| - if is_enabled - gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) - else - rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - end + wrapped_gitaly_errors do + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) end end @@ -61,57 +55,6 @@ module Gitlab def gitaly_conflicts_client(repository) repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) end - - def write_resolved_file_to_index(repository, index, file, params) - if params[:sections] - resolved_lines = file.resolve_lines(params[:sections]) - new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - - new_file << "\n" if file.our_blob.data.end_with?("\n") - elsif params[:content] - new_file = file.resolve_content(params[:content]) - end - - our_path = file.our_path - - oid = repository.rugged.write(new_file, :blob) - index.add(path: our_path, oid: oid, mode: file.our_mode) - index.conflict_remove(our_path) - end - - def rugged_list_conflict_files - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) - end - - def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - resolution.files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" - end - - commit_params = { - message: resolution.commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(resolution.user, source_branch, index, commit_params) - end - end end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index b58296375ef..74a4633424f 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,6 +1,5 @@ -# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. +# frozen_string_literal: true -# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git class Diff @@ -22,13 +21,17 @@ module Gitlab alias_method :expanded?, :expanded - SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + # The default maximum content size to display a diff patch. + # + # If this value ever changes, make sure to create a migration to update + # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. + DEFAULT_MAX_PATCH_BYTES = 100.kilobytes - # The maximum size of a diff to display. - SIZE_LIMIT = 100.kilobytes + # This is a limitation applied on the source (Gitaly), therefore we don't allow + # persisting limits over that. + MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - # The maximum size before a diff is collapsed. - COLLAPSE_LIMIT = 10.kilobytes + SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze class << self def between(repo, head, base, options = {}, *paths) @@ -52,20 +55,31 @@ module Gitlab repo.diff(common_commit, head, actual_options, *paths) end - # Return a copy of the +options+ hash containing only keys that can be - # passed to Rugged. Allowed options are: + # Return a copy of the +options+ hash containing only recognized keys. + # Allowed options are: # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :disable_pathspec_match :: - # If true, the given +*paths+ will be applied as exact matches, - # instead of as fnmatch patterns. + # :max_files :: + # Limit how many files will patches be allowed for before collapsing + # + # :max_lines :: + # Limit how many patch lines (across all files) will be allowed for + # before collapsing # + # :limits :: + # A hash with additional limits to check before collapsing patches. + # Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines` + # and `safe_max_bytes` + # + # :expanded :: + # If false, patch raw data will not be included in the diff after + # `max_files`, `max_lines` or any of the limits in `limits` are + # exceeded def filter_diff_options(options, default_options = {}) - allowed_options = [:ignore_whitespace_change, - :disable_pathspec_match, :paths, - :max_files, :max_lines, :limits, :expanded] + allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, + :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -93,10 +107,30 @@ module Gitlab # # "Binary files a/file/path and b/file/path differ\n" # This is used when we detect that a diff is binary - # using CharlockHolmes when Rugged treats it as text. + # using CharlockHolmes. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end + + # Returns the limit of bytes a single diff file can reach before it + # appears as 'collapsed' for end-users. + # By convention, it's 10% of the persisted `diff_max_patch_bytes`. + # + # Example: If we have 100k for the `diff_max_patch_bytes`, it will be 10k by + # default. + # + # Patches surpassing this limit should still be persisted in the database. + def patch_safe_limit_bytes + patch_hard_limit_bytes / 10 + end + + # Returns the limit for a single diff file (patch). + # + # Patches surpassing this limit shouldn't be persisted in the database + # and will be presented as 'too large' for end-users. + def patch_hard_limit_bytes + Gitlab::CurrentSettings.diff_max_patch_bytes + end end def initialize(raw_diff, expanded: true) @@ -106,8 +140,6 @@ module Gitlab when Hash init_from_hash(raw_diff) prune_diff_if_eligible - when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff) when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible @@ -144,7 +176,7 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -162,7 +194,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT + @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes end def collapse! @@ -184,31 +216,6 @@ module Gitlab private - def init_from_rugged(rugged) - if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged) - d = rugged.delta - else - d = rugged - end - - @new_path = encode!(d.new_file[:path]) - @old_path = encode!(d.old_file[:path]) - @a_mode = d.old_file[:mode].to_s(8) - @b_mode = d.new_file[:mode].to_s(8) - @new_file = d.added? - @renamed_file = d.renamed? - @deleted_file = d.deleted? - end - - def init_from_rugged_patch(patch) - # Don't bother initializing diffs that are too large. If a diff is - # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch) - - @diff = encode!(strip_diff_headers(patch.to_s)) - end - def init_from_hash(hash) raw_diff = hash.symbolize_keys @@ -226,6 +233,7 @@ module Gitlab @new_file = diff.from_id == BLANK_SHA @renamed_file = diff.from_path != diff.to_path @deleted_file = diff.to_id == BLANK_SHA + @too_large = diff.too_large if diff.respond_to?(:too_large) collapse! if diff.respond_to?(:collapsed) && diff.collapsed end @@ -237,47 +245,6 @@ module Gitlab collapse! end end - - # If the patch surpasses any of the diff limits it calls the appropiate - # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch) - size = 0 - - patch.each_hunk do |hunk| - hunk.each_line do |line| - size += line.content.bytesize - - if size >= SIZE_LIMIT - too_large! - return true # rubocop:disable Cop/AvoidReturnFromBlocks - end - end - end - - if !expanded && size >= COLLAPSE_LIMIT - collapse! - return true - end - - false - end - - # Strip out the information at the beginning of the patch's text to match - # Grit's output - def strip_diff_headers(diff_text) - # Delete everything up to the first line that starts with '---' or - # 'Binary' - diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') - - if diff_text.start_with?('---', 'Binary') - diff_text - else - # If the diff_text did not contain a line starting with '---' or - # 'Binary', return the empty string. No idea why; we are just - # preserving behavior from before the refactor. - '' - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 6a601561c2a..5c70cb6c66c 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab @@ -11,7 +13,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.collection_limits(options = {}) + def self.limits(options = {}) limits = {} limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) @@ -19,13 +21,14 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes OpenStruct.new(limits) end def initialize(iterator, options = {}) @iterator = iterator - @limits = self.class.collection_limits(options) + @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) @@ -42,12 +45,10 @@ module Gitlab return if @overflow return if @iterator.nil? - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled && @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) - each_gitaly_patch(&block) - else - each_rugged_patch(&block) - end + if @iterator.is_a?(Gitlab::GitalyClient::DiffStitcher) + each_gitaly_patch(&block) + else + each_serialized_patch(&block) end @populated = true @@ -118,7 +119,7 @@ module Gitlab end end - def each_rugged_patch + def each_serialized_patch i = @array.length @iterator.each do |raw| diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb new file mode 100644 index 00000000000..998c41497a2 --- /dev/null +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class DiffStatsCollection + include Gitlab::Utils::StrongMemoize + include Enumerable + + def initialize(diff_stats) + @collection = diff_stats + end + + def each(&block) + @collection.each(&block) + end + + def find_by_path(path) + indexed_by_path[path] + end + + def paths + @collection.map(&:path) + end + + private + + def indexed_by_path + strong_memoize(:indexed_by_path) do + index_by { |stats| stats.path } + end + end + end + end +end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 00c943fdb25..00000000000 --- a/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,285 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - Gitlab::GitalyClient.migrate(:import_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_import_repository(source) - else - git_import_repository(source, timeout) - end - end - end - - def fork_repository(new_shard_name, new_repository_relative_path) - Gitlab::GitalyClient.migrate(:fork_repository, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_fork_repository(new_shard_name, new_repository_relative_path) - else - git_fork_repository(new_shard_name, new_repository_relative_path) - end - end - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option - - setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - - success - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def gitaly_import_repository(source) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) - true - rescue GRPC::BadStatus => e - @output << e.message - false - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - - def gitaly_fork_repository(new_shard_name, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) - raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) - - Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) - rescue GRPC::BadStatus => e - logger.error "fork-repository failed: #{e.message}" - false - end - end - end -end diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 4b505312f60..575e12390cd 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb deleted file mode 100644 index 7c201c6169b..00000000000 --- a/lib/gitlab/git/hook.rb +++ /dev/null @@ -1,108 +0,0 @@ -# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in -# app/services. We shouldn't bother migrating this until we know how -# Gitlab::Git::HooksService will be migrated. - -module Gitlab - module Git - class Hook - GL_PROTOCOL = 'web'.freeze - attr_reader :name, :path, :repository - - def initialize(name, repository) - @name = name - @repository = repository - @path = File.join(repo_path.strip, 'hooks', name) - end - - def repo_path - repository.path - end - - def exists? - File.exist?(path) - end - - def trigger(gl_id, gl_username, oldrev, newrev, ref) - return [true, nil] unless exists? - - Bundler.with_clean_env do - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - end - end - end - - private - - def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - changes = [oldrev, newrev, ref].join(" ") - - exit_status = false - exit_message = nil - - vars = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => repository.gl_repository - } - - options = { - chdir: repo_path - } - - Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| - exit_status = true - stdin.sync = true - - # in git, pre- and post- receive hooks may just exit without - # reading stdin. We catch the exception to avoid a broken pipe - # warning - begin - # inject all the changes as stdin to the hook - changes.lines do |line| - stdin.puts line - end - rescue Errno::EPIPE - end - - stdin.close - - unless wait_thr.value == 0 - exit_status = false - exit_message = retrieve_error_message(stderr, stdout) - end - end - - [exit_status, exit_message] - end - - def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path - } - - options = { - chdir: repo_path - } - - args = [ref, oldrev, newrev] - - stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, Gitlab::Utils.nlbr(stderr.presence || stdout)] - end - - def retrieve_error_message(stderr, stdout) - err_message = stderr.read - err_message = err_message.blank? ? stdout.read : err_message - Gitlab::Utils.nlbr(err_message) - end - end - end -end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index 455e8451c10..892a069a3b7 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab @@ -17,18 +19,18 @@ module Gitlab ].freeze def self.set(gl_repository, env) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? raise "missing gl_repository" if gl_repository.blank? - RequestStore.store[:gitlab_git_env] ||= {} - RequestStore.store[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) end def self.all(gl_repository) - return {} unless RequestStore.active? + return {} unless Gitlab::SafeRequestStore.active? - h = RequestStore.fetch(:gitlab_git_env) { {} } + h = Gitlab::SafeRequestStore.fetch(:gitlab_git_env) { {} } h.fetch(gl_repository, {}) end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb deleted file mode 100644 index f302b852b35..00000000000 --- a/lib/gitlab/git/hooks_service.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module Git - class HooksService - PreReceiveError = Class.new(StandardError) - - attr_accessor :oldrev, :newrev, :ref - - def execute(pusher, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = pusher.gl_id - @gl_username = pusher.username - @oldrev = oldrev - @newrev = newrev - @ref = ref - - %w(pre-receive update).each do |hook_name| - status, message = run_hook(hook_name) - - unless status - raise PreReceiveError, message - end - end - - yield(self).tap do - run_hook('post-receive') - end - end - - private - - def run_hook(name) - hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) - end - end - end -end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index d94082a3e30..3b9b516308f 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,157 +1,9 @@ -# Gitaly note: JV: When the time comes I think we will want to copy this -# class into Gitaly. None of its methods look like they should be RPC's. -# The RPC's will be at a higher level. +# frozen_string_literal: true module Gitlab module Git class Index IndexError = Class.new(StandardError) - - DEFAULT_MODE = 0o100644 - - ACTIONS = %w(create create_dir update move delete).freeze - ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze - - attr_reader :repository, :raw_index - - def initialize(repository) - @repository = repository - @raw_index = repository.rugged.index - end - - delegate :read_tree, :get, to: :raw_index - - def apply(action, options) - validate_action!(action) - public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend - end - - def write_tree - raw_index.write_tree(repository.rugged) - end - - def dir_exists?(path) - raw_index.find { |entry| entry[:path].start_with?("#{path}/") } - end - - def create(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - add_blob(options) - end - - def create_dir(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - if dir_exists?(options[:file_path]) - raise IndexError, "A directory with this name already exists" - end - - options = options.dup - options[:file_path] += '/.gitkeep' - options[:content] = '' - - add_blob(options) - end - - def update(options) - options = normalize_options(options) - - file_entry = get(options[:file_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - add_blob(options, mode: file_entry[:mode]) - end - - def move(options) - options = normalize_options(options) - - file_entry = get(options[:previous_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - raw_index.remove(options[:previous_path]) - - add_blob(options, mode: file_entry[:mode]) - end - - def delete(options) - options = normalize_options(options) - - unless get(options[:file_path]) - raise IndexError, "A file with this name doesn't exist" - end - - raw_index.remove(options[:file_path]) - end - - private - - def normalize_options(options) - options = options.dup - options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] - options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] - options - end - - def normalize_path(path) - unless path - raise IndexError, "You must provide a file path" - end - - pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - - pathname.each_filename do |segment| - if segment == '..' - raise IndexError, 'Path cannot include directory traversal' - end - end - - pathname.to_s - end - - def add_blob(options, mode: nil) - content = options[:content] - unless content - raise IndexError, "You must provide content" - end - - content = Base64.decode64(content) if options[:encoding] == 'base64' - - detect = CharlockHolmes::EncodingDetector.new.detect(content) - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repository.rugged.write(content, :blob) - - raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) - rescue Rugged::IndexError => e - raise IndexError, e.message - end - - def validate_action!(action) - unless ACTIONS.include?(action.to_s) - raise ArgumentError, "Unknown action '#{action}'" - end - end end end end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index b9e5cf258f4..8e2a925dfea 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git class LfsChanges @@ -6,46 +8,12 @@ module Gitlab @newrev = newrev end - def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) - else - git_new_pointers(object_limit, not_in) - end - end + def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil) + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout) end def all_pointers - @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) - else - git_all_pointers - end - end - end - - private - - def git_new_pointers(object_limit, not_in) - @new_pointers ||= begin - rev_list.new_objects(not_in: not_in, require_path: true) do |object_ids| - object_ids = object_ids.take(object_limit) if object_limit - - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - end - - def git_all_pointers - rev_list.all_objects(require_path: true) do |object_ids| - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - - def rev_list - Gitlab::Git::RevList.new(@repository, newrev: @newrev) + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) end end end diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb index 2ae0a889590..b7019a221ac 100644 --- a/lib/gitlab/git/lfs_pointer_file.rb +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git class LfsPointerFile diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb new file mode 100644 index 00000000000..b27f7038c26 --- /dev/null +++ b/lib/gitlab/git/merge_base.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class MergeBase + include Gitlab::Utils::StrongMemoize + + def initialize(repository, refs) + @repository, @refs = repository, refs + end + + # Returns the SHA of the first common ancestor + def sha + if unknown_refs.any? + raise UnknownRef, "Can't find merge base for unknown refs: #{unknown_refs.inspect}" + end + + strong_memoize(:sha) do + @repository.merge_base(*commits_for_refs) + end + end + + # Returns the merge base as a Gitlab::Git::Commit + def commit + return unless sha + + @commit ||= @repository.commit_by(oid: sha) + end + + # Returns the refs passed on initialization that aren't found in + # the repository, and thus cannot be used to find a merge base. + def unknown_refs + @unknown_refs ||= Hash[@refs.zip(commits_for_refs)] + .select { |ref, commit| commit.nil? }.keys + end + + private + + def commits_for_refs + @commits_for_refs ||= @repository.commits_by(oids: @refs) + end + end + end +end diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb new file mode 100644 index 00000000000..1c6242b444a --- /dev/null +++ b/lib/gitlab/git/object_pool.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class ObjectPool + # GL_REPOSITORY has to be passed for Gitlab::Git::Repositories, but not + # used for ObjectPools. + GL_REPOSITORY = "" + + delegate :exists?, :size, to: :repository + delegate :unlink_repository, :delete, to: :object_pool_service + + attr_reader :storage, :relative_path, :source_repository + + def initialize(storage, relative_path, source_repository) + @storage = storage + @relative_path = relative_path + @source_repository = source_repository + end + + def create + object_pool_service.create(source_repository) + end + + def link(to_link_repo) + object_pool_service.link_repository(to_link_repo) + end + + def gitaly_object_pool + Gitaly::ObjectPool.new(repository: to_gitaly_repository) + end + + def to_gitaly_repository + Gitlab::GitalyClient::Util.repository(storage, relative_path, GL_REPOSITORY) + end + + # Allows for reusing other RPCs by 'tricking' Gitaly to think its a repository + def repository + @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY) + end + + private + + def object_pool_service + @object_pool_service ||= Gitlab::GitalyClient::ObjectPoolService.new(self) + end + + def relative_path_to(pool_member_path) + pool_path = Pathname.new("#{relative_path}#{File::SEPARATOR}") + + Pathname.new(pool_member_path).relative_path_from(pool_path).to_s + end + end + end +end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 280def182d5..8797d3dce24 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,13 +1,15 @@ +# frozen_string_literal: true + module Gitlab module Git class OperationService - include Gitlab::Git::Popen - BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created def self.from_gitaly(branch_update) + return if branch_update.nil? + new( branch_update.commit_id, branch_update.repo_created, @@ -15,177 +17,6 @@ module Gitlab ) end end - - attr_reader :user, :repository - - def initialize(user, new_repository) - if user - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - @user = user - end - - # Refactoring aid - Gitlab::Git.check_namespace!(new_repository) - - @repository = new_repository - end - - def add_branch(branch_name, newrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - oldrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def rm_branch(branch) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name - oldrev = branch.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def add_tag(tag_name, newrev, options = {}) - ref = Gitlab::Git::TAG_REF_PREFIX + tag_name - oldrev = Gitlab::Git::BLANK_SHA - - with_hooks(ref, newrev, oldrev) do |service| - # We want to pass the OID of the tag object to the hooks. For an - # annotated tag we don't know that OID until after the tag object - # (raw_tag) is created in the repository. That is why we have to - # update the value after creating the tag object. Only the - # "post-receive" hook will receive the correct value in this case. - raw_tag = repository.rugged.tags.create(tag_name, newrev, options) - service.newrev = raw_tag.target_id - end - end - - def rm_tag(tag) - ref = Gitlab::Git::TAG_REF_PREFIX + tag.name - oldrev = tag.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) do - repository.rugged.tags.delete(tag_name) - end - end - - # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, - # it would be created from `start_branch_name`. - # If `start_repository` is passed, and the branch doesn't exist, - # it would try to find the commits from it instead of current repository. - def with_branch( - branch_name, - start_branch_name: nil, - start_repository: repository, - &block) - - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - start_branch_name = nil if start_repository.empty? - - if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" - end - - update_branch_with_hooks(branch_name) do - repository.with_repo_branch_commit( - start_repository, - start_branch_name || branch_name, - &block) - end - end - - def update_branch(branch_name, newrev, oldrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - end - - private - - # Returns [newrev, should_run_after_create, should_run_after_create_branch] - def update_branch_with_hooks(branch_name) - update_autocrlf_option - - was_empty = repository.empty? - - # Make commit - newrev = yield - - unless newrev - raise Gitlab::Git::CommitError.new('Failed to create commit') - end - - branch = repository.find_branch(branch_name) - oldrev = find_oldrev_from_branch(newrev, branch) - - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - - BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) - end - - def find_oldrev_from_branch(newrev, branch) - return Gitlab::Git::BLANK_SHA unless branch - - oldrev = branch.target - - merge_base = repository.merge_base(newrev, branch.target) - raise Gitlab::Git::Repository::InvalidRef unless merge_base - - if oldrev == merge_base - oldrev - else - raise Gitlab::Git::CommitError.new('Branch diverged') - end - end - - def update_ref_in_hooks(ref, newrev, oldrev) - with_hooks(ref, newrev, oldrev) do - update_ref(ref, newrev, oldrev) - end - end - - def with_hooks(ref, newrev, oldrev) - Gitlab::Git::HooksService.new.execute( - user, - repository, - oldrev, - newrev, - ref) do |service| - - yield(service) - end - end - - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. - def update_ref(ref, newrev, oldrev) - # We use 'git update-ref' because libgit2/rugged currently does not - # offer 'compare and swap' ref updates. Without compare-and-swap we can - # (and have!) accidentally reset the ref to an earlier state, clobbering - # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - - output, status = popen( - command, - repository.path) do |stdin| - stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") - end - - unless status.zero? - Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") - raise Gitlab::Git::CommitError.new( - "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ - " Please refresh and try again.") - end - end - - def update_autocrlf_option - if repository.autocrlf != :input - repository.autocrlf = :input - end - end end end end diff --git a/lib/gitlab/git/patches/collection.rb b/lib/gitlab/git/patches/collection.rb new file mode 100644 index 00000000000..ad6b5d32abc --- /dev/null +++ b/lib/gitlab/git/patches/collection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Collection + MAX_PATCH_SIZE = 2.megabytes + + def initialize(one_or_more_patches) + @patches = Array(one_or_more_patches).map do |patch_content| + Gitlab::Git::Patches::Patch.new(patch_content) + end + end + + def content + @patches.map(&:content).join("\n") + end + + def valid_size? + size < MAX_PATCH_SIZE + end + + # rubocop: disable CodeReuse/ActiveRecord + # `@patches` is not an `ActiveRecord` relation, but an `Enumerable` + # We're using sum from `ActiveSupport` + def size + @size ||= @patches.sum(&:size) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb new file mode 100644 index 00000000000..c62994432d3 --- /dev/null +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class CommitPatches + include Gitlab::Git::WrapsGitalyErrors + + def initialize(user, repository, branch, patch_collection) + @user, @repository, @branch, @patches = user, repository, branch, patch_collection + end + + def commit + repository.with_cache_hooks do + wrapped_gitaly_errors do + operation_service.user_commit_patches(user, branch, patches.content) + end + end + end + + private + + attr_reader :user, :repository, :branch, :patches + + def operation_service + repository.raw.gitaly_operation_client + end + end + end + end +end diff --git a/lib/gitlab/git/patches/patch.rb b/lib/gitlab/git/patches/patch.rb new file mode 100644 index 00000000000..fe6ae1b5b00 --- /dev/null +++ b/lib/gitlab/git/patches/patch.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Patch + attr_reader :content + + def initialize(content) + @content = content + end + + def size + content.bytesize + end + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 57b82a37d6c..e3a2031eeca 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb deleted file mode 100644 index f9f24ecc48d..00000000000 --- a/lib/gitlab/git/popen.rb +++ /dev/null @@ -1,104 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -require 'open3' - -module Gitlab - module Git - module Popen - FAST_GIT_PROCESS_TIMEOUT = 15.seconds - - def popen(cmd, path, vars = {}, lazy_block: nil) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - options = { chdir: path } - - cmd_output = "" - cmd_status = 0 - Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - stdout.set_encoding(Encoding::ASCII_8BIT) - - yield(stdin) if block_given? - stdin.close - - if lazy_block - cmd_output = lazy_block.call(stdout.lazy) - cmd_status = 0 - break - else - cmd_output << stdout.read - end - - cmd_output << stderr.read - cmd_status = wait_thr.value.exitstatus - end - - [cmd_output, cmd_status] - end - - def popen_with_timeout(cmd, timeout, path, vars = {}) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - - unless File.directory?(path) - FileUtils.mkdir_p(path) - end - - rout, wout = IO.pipe - rerr, werr = IO.pipe - - pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) - - begin - status = process_wait_with_timeout(pid, timeout) - - # close write ends so we could read them - wout.close - werr.close - - cmd_output = rout.readlines.join - cmd_output << rerr.readlines.join # Copying the behaviour of `popen` which merges stderr into output - - [cmd_output, status.exitstatus] - rescue Timeout::Error => e - kill_process_group_for_pid(pid) - - raise e - ensure - wout.close unless wout.closed? - werr.close unless werr.closed? - - rout.close - rerr.close - end - end - - def process_wait_with_timeout(pid, timeout) - deadline = timeout.seconds.from_now - wait_time = 0.01 - - while deadline > Time.now - sleep(wait_time) - _, status = Process.wait2(pid, Process::WNOHANG) - - return status unless status.nil? - end - - raise Timeout::Error, "Timeout waiting for process ##{pid}" - end - - def kill_process_group_for_pid(pid) - Process.kill("KILL", -pid) - Process.wait(pid) - rescue Errno::ESRCH - end - end - end -end diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb new file mode 100644 index 00000000000..03caace6fce --- /dev/null +++ b/lib/gitlab/git/pre_receive_error.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Git + # + # PreReceiveError is special because its message gets displayed to users + # in the web UI. To prevent XSS we sanitize the message on + # initialization. + class PreReceiveError < StandardError + def initialize(msg = '') + super(nlbr(msg)) + end + + private + + # In gitaly-ruby we override this method to do nothing, so that + # sanitization happens in gitlab-rails only. + def nlbr(str) + Gitlab::Utils.nlbr(str) + end + end + end +end diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb new file mode 100644 index 00000000000..b6577ba17f1 --- /dev/null +++ b/lib/gitlab/git/push.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class Push + include Gitlab::Utils::StrongMemoize + + attr_reader :ref, :oldrev, :newrev + + def initialize(project, oldrev, newrev, ref) + @project = project + @oldrev = oldrev.presence || Gitlab::Git::BLANK_SHA + @newrev = newrev.presence || Gitlab::Git::BLANK_SHA + @ref = ref + end + + def branch_name + strong_memoize(:branch_name) do + Gitlab::Git.branch_name(@ref) + end + end + + def branch_added? + Gitlab::Git.blank_ref?(@oldrev) + end + + def branch_removed? + Gitlab::Git.blank_ref?(@newrev) + end + + def branch_updated? + branch_push? && !branch_added? && !branch_removed? + end + + def force_push? + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end + + def branch_push? + strong_memoize(:branch_push) do + Gitlab::Git.branch_ref?(@ref) + end + end + + def modified_paths + unless branch_updated? + raise ArgumentError, 'Unable to calculate modified paths!' + end + + strong_memoize(:modified_paths) do + @project.repository.diff_stats(@oldrev, @newrev).paths + end + end + end + end +end diff --git a/lib/gitlab/git/raw_diff_change.rb b/lib/gitlab/git/raw_diff_change.rb index 98de9328071..e1002af40f6 100644 --- a/lib/gitlab/git/raw_diff_change.rb +++ b/lib/gitlab/git/raw_diff_change.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git # This class behaves like a struct with fields :blob_id, :blob_size, :operation, :old_path, :new_path diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index fa71a4e7ea7..eec91194949 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,4 +1,4 @@ -# Gitaly note: JV: probably no RPC's here (just one interaction with Rugged). +# frozen_string_literal: true module Gitlab module Git @@ -26,13 +26,6 @@ module Gitlab str.gsub(%r{\Arefs/heads/}, '') end - # Gitaly: this method will probably be migrated indirectly via its call sites. - def self.dereference_object(object) - object = object.target while object.is_a?(Rugged::Tag::Annotation) - - object - end - def initialize(repository, name, target, dereferenced_target) @name = Gitlab::Git.ref_name(name) @dereferenced_target = dereferenced_target diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index ebe46722890..df3cd422527 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -1,87 +1,28 @@ +# frozen_string_literal: true + module Gitlab module Git class RemoteMirror - def initialize(repository, ref_name) - @repository = repository - @ref_name = ref_name - end - - def update(only_branches_matching: []) - @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| - if is_enabled - gitaly_update(only_branches_matching) - else - rugged_update(only_branches_matching) - end - end - end - - private - - def gitaly_update(only_branches_matching) - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end + include Gitlab::Git::WrapsGitalyErrors - def rugged_update(only_branches_matching) - local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) - remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + attr_reader :repository, :ref_name, :only_branches_matching, :ssh_key, :known_hosts - updated_branches = changed_refs(local_branches, remote_branches) - push_branches(updated_branches.keys) if updated_branches.present? - - delete_refs(local_branches, remote_branches) - - local_tags = refs_obj(@repository.tags) - remote_tags = refs_obj(@repository.remote_tags(@ref_name)) - - updated_tags = changed_refs(local_tags, remote_tags) - @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? - - delete_refs(local_tags, remote_tags) - end - - def refs_obj(refs, only_refs_matching: []) - refs.each_with_object({}) do |ref, refs| - next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) - - refs[ref.name] = ref - end - end - - def changed_refs(local_refs, remote_refs) - local_refs.select do |ref_name, ref| - remote_ref = remote_refs[ref_name] - - remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target - end - end - - def push_branches(branches) - default_branch, branches = branches.partition do |branch| - @repository.root_ref == branch - end - - # Push the default branch first so it works fine when remote mirror is empty. - branches.unshift(*default_branch) - - @repository.push_remote_branches(@ref_name, branches) - end - - def delete_refs(local_refs, remote_refs) - refs = refs_to_delete(local_refs, remote_refs) - - @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? - end - - def refs_to_delete(local_refs, remote_refs) - default_branch_id = @repository.commit.id - - remote_refs.select do |remote_ref_name, remote_ref| - next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo - - remote_ref_id = remote_ref.dereferenced_target.try(:id) - - remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + def initialize(repository, ref_name, only_branches_matching: [], ssh_key: nil, known_hosts: nil) + @repository = repository + @ref_name = ref_name + @only_branches_matching = only_branches_matching + @ssh_key = ssh_key + @known_hosts = known_hosts + end + + def update + wrapped_gitaly_errors do + repository.gitaly_remote_client.update_remote_mirror( + ref_name, + only_branches_matching, + ssh_key: ssh_key, + known_hosts: known_hosts + ) end end end diff --git a/lib/gitlab/git/remote_repository.rb b/lib/gitlab/git/remote_repository.rb index f40e59a8dd0..234541d8145 100644 --- a/lib/gitlab/git/remote_repository.rb +++ b/lib/gitlab/git/remote_repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git # diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4cbf20bfe76..786c90f9272 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1,4 +1,5 @@ -# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object +# frozen_string_literal: true + require 'tempfile' require 'forwardable' require "rubygems/package" @@ -7,19 +8,12 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen + include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize - ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY - GIT_ALTERNATE_OBJECT_DIRECTORIES - ].freeze - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze SEARCH_CONTEXT_LINES = 3 + REV_LIST_COMMIT_LIMIT = 2_000 # In https://gitlab.com/gitlab-org/gitaly/merge_requests/698 # We copied these two prefixes into gitaly-go, so don't change these # or things will break! (REBASE_WORKTREE_PREFIX and SQUASH_WORKTREE_PREFIX) @@ -40,19 +34,6 @@ module Gitlab ChecksumError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the repository path - def create(repo_path, bare: true, symlink_hooks_to: nil) - FileUtils.mkdir_p(repo_path, mode: 0770) - - # Equivalent to `git --git-path=#{repo_path} init [--bare]` - repo = Rugged::Repository.init_at(repo_path, bare) - repo.close - - create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present? - - true - end - def create_hooks(repo_path, global_hooks_path) local_hooks_path = File.join(repo_path, 'hooks') real_local_hooks_path = :not_found @@ -86,10 +67,14 @@ module Gitlab # Relative path of repo attr_reader :relative_path - # Rugged repo object - attr_reader :rugged + attr_reader :storage, :gl_repository, :relative_path - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + # This remote name has to be stable for all types of repositories that + # can join an object pool. If it's structure ever changes, a migration + # has to be performed on the object pools to update the remote names. + # Else the pool can't be updated anymore and is left in an inconsistent + # state. + alias_method :object_pool_remote_name, :gl_repository # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -98,51 +83,33 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - @name = @relative_path.split("/").last end def ==(other) - path == other.path + other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end + alias_method :eql?, :== + + def hash + [self.class, storage, relative_path].hash + end + + # This method will be removed when Gitaly reaches v1.1. def path - @path ||= File.join( + File.join( Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path ) end # Default branch in the repository def root_ref - @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| - if is_enabled - gitaly_ref_client.default_branch_name - else - discover_default_branch - end - end - end - - def rugged - @rugged ||= circuit_breaker.perform do - Rugged::Repository.new(path, alternates: alternate_object_directories) - end - rescue Rugged::RepositoryError, Rugged::OSError - raise NoRepository.new('no repository for such path') - end - - def cleanup - @rugged&.close - end - - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) + gitaly_ref_client.default_branch_name + rescue GRPC::NotFound => e + raise NoRepository.new(e.message) + rescue GRPC::Unknown => e + raise Gitlab::Git::CommandError.new(e.message) end def exists? @@ -152,79 +119,36 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - gitaly_migrate(:branch_names) do |is_enabled| - if is_enabled - gitaly_ref_client.branch_names - else - branches.map(&:name) - end + wrapped_gitaly_errors do + gitaly_ref_client.branch_names end end # Returns an Array of Branches def branches - gitaly_migrate(:branches) do |is_enabled| - if is_enabled - gitaly_ref_client.branches - else - branches_filter - end + wrapped_gitaly_errors do + gitaly_ref_client.branches end end - def reload_rugged - @rugged = nil - end - # Directly find a branch with a simple name (e.g. master) # - # force_reload causes a new Rugged repository to be instantiated - # - # This is to work around a bug in libgit2 that causes in-memory refs to - # be stale/invalid when packed-refs is changed. - # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 - def find_branch(name, force_reload = false) - gitaly_migrate(:find_branch) do |is_enabled| - if is_enabled - gitaly_ref_client.find_branch(name) - else - reload_rugged if force_reload - - rugged_ref = rugged.branches[name] - if rugged_ref - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - end - end + def find_branch(name) + wrapped_gitaly_errors do + gitaly_ref_client.find_branch(name) end end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches) do |is_enabled| - if is_enabled - gitaly_ref_client.local_branches(sort_by: sort_by) - else - branches_filter(filter: :local, sort_by: sort_by) - end + wrapped_gitaly_errors do + gitaly_ref_client.local_branches(sort_by: sort_by) end end # Returns the number of valid branches def branch_count - gitaly_migrate(:branch_names) do |is_enabled| - if is_enabled - gitaly_ref_client.count_branch_names - else - rugged.branches.each(:local).count do |ref| - begin - ref.name && ref.target # ensures the branch is valid - - true - rescue Rugged::ReferenceError - false - end - end - end + wrapped_gitaly_errors do + gitaly_ref_client.count_branch_names end end @@ -245,50 +169,25 @@ module Gitlab # This refs by default not visible in project page and not cloned to client side. alias_method :has_visible_content?, :has_local_branches? - def has_local_branches_rugged? - rugged.branches.each(:local).any? do |ref| - begin - ref.name && ref.target # ensures the branch is valid - - true - rescue Rugged::ReferenceError - false - end - end - end - # Returns the number of valid tags def tag_count - gitaly_migrate(:tag_names) do |is_enabled| - if is_enabled - gitaly_ref_client.count_tag_names - else - rugged.tags.count - end + wrapped_gitaly_errors do + gitaly_ref_client.count_tag_names end end # Returns an Array of tag names def tag_names - gitaly_migrate(:tag_names) do |is_enabled| - if is_enabled - gitaly_ref_client.tag_names - else - rugged.tags.map { |t| t.name } - end + wrapped_gitaly_errors do + gitaly_ref_client.tag_names end end # Returns an Array of Tags # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags - gitaly_migrate(:tags) do |is_enabled| - if is_enabled - tags_from_gitaly - else - tags_from_rugged - end + wrapped_gitaly_errors do + gitaly_ref_client.tags end end @@ -296,13 +195,8 @@ module Gitlab # # Ref names must start with `refs/`. def ref_exists?(ref_name) - gitaly_migrate(:ref_exists, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_exists?(ref_name) - else - rugged_ref_exists?(ref_name) - end + wrapped_gitaly_errors do + gitaly_ref_exists?(ref_name) end end @@ -310,12 +204,8 @@ module Gitlab # # name - The name of the tag as a String. def tag_exists?(name) - gitaly_migrate(:ref_exists_tags) do |is_enabled| - if is_enabled - gitaly_ref_exists?("refs/tags/#{name}") - else - rugged_tag_exists?(name) - end + wrapped_gitaly_errors do + gitaly_ref_exists?("refs/tags/#{name}") end end @@ -323,20 +213,8 @@ module Gitlab # # name - The name of the branch as a String. def branch_exists?(name) - gitaly_migrate(:ref_exists_branches) do |is_enabled| - if is_enabled - gitaly_ref_exists?("refs/heads/#{name}") - else - rugged_branch_exists?(name) - end - end - end - - def batch_existence(object_ids, existing: true) - filter_method = existing ? :select : :reject - - object_ids.public_send(filter_method) do |oid| # rubocop:disable GitlabSecurity/PublicSend - rugged.exists?(oid) + wrapped_gitaly_errors do + gitaly_ref_exists?("refs/heads/#{name}") end end @@ -346,61 +224,17 @@ module Gitlab end def delete_all_refs_except(prefixes) - gitaly_migrate(:ref_delete_refs) do |is_enabled| - if is_enabled - gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) - else - delete_refs(*all_ref_names_except(prefixes)) - end + wrapped_gitaly_errors do + gitaly_ref_client.delete_refs(except_with_prefixes: prefixes) end end - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - - # Discovers the default branch based on the repository's available branches - # - # - If no branches are present, returns nil - # - If one branch is present, returns its name - # - If two or more branches are present, returns current HEAD or master or first branch - def discover_default_branch - names = branch_names - - return if names.empty? - - return names[0] if names.length == 1 - - if rugged_head - extracted_name = Ref.extract_branch_name(rugged_head.name) - - return extracted_name if names.include?(extracted_name) - end - - if names.include?('master') - 'master' - else - names[0] - end - end - - def rugged_head - rugged.head - rescue Rugged::ReferenceError - nil - end - - def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) + def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id, append_sha: append_sha) + prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) { 'ArchivePrefix' => prefix, @@ -412,16 +246,12 @@ module Gitlab # This is both the filename of the archive (missing the extension) and the # name of the top-level member of the archive under which all files go - # - # FIXME: The generated prefix is incorrect for projects with hashed - # storage enabled - def archive_prefix(ref, sha, append_sha:) + def archive_prefix(ref, sha, project_path, append_sha:) append_sha = (ref != sha) if append_sha.nil? - project_name = self.name.chomp('.git') formatted_ref = ref.tr('/', '-') - prefix_segments = [project_name, formatted_ref] + prefix_segments = [project_path, formatted_ref] prefix_segments << sha if append_sha prefix_segments.join('-') @@ -466,18 +296,12 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end - # Use the Rugged Walker API to build an array of commits. + # Build an array of commits. # # Usage. # repo.log( @@ -487,8 +311,6 @@ module Gitlab # offset: 5, # after: Time.new(2016, 4, 21, 14, 32, 10) # ) - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446 def log(options) default_options = { limit: 10, @@ -509,69 +331,45 @@ module Gitlab raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") end - gitaly_migrate(:find_commits) do |is_enabled| - if is_enabled - gitaly_commit_client.find_commits(options) - else - raw_log(options).map { |c| Commit.decorate(self, c) } - end + wrapped_gitaly_errors do + gitaly_commit_client.find_commits(options) end end - # Used in gitaly-ruby - def raw_log(options) - sha = - unless options[:all] - actual_ref = options[:ref] || root_ref - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - end - - log_by_shell(sha, options) + def new_commits(newrev) + wrapped_gitaly_errors do + gitaly_ref_client.list_new_commits(newrev) + end end - def count_commits(options) - count_commits_options = process_count_commits_options(options) + def new_blobs(newrev) + return [] if newrev.blank? || newrev == ::Gitlab::Git::BLANK_SHA - gitaly_migrate(:count_commits) do |is_enabled| - if is_enabled - count_commits_by_gitaly(count_commits_options) - else - count_commits_by_shelling_out(count_commits_options) + strong_memoize("new_blobs_#{newrev}") do + wrapped_gitaly_errors do + gitaly_ref_client.list_new_blobs(newrev, REV_LIST_COMMIT_LIMIT) end end end - # Return the object that +revspec+ points to. If +revspec+ is an - # annotated tag, then return the tag's target instead. - def rev_parse_target(revspec) - obj = rugged.rev_parse(revspec) - Ref.dereference_object(obj) - end - - # Return a collection of Rugged::Commits between the two revspec arguments. - # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for - # a detailed list of valid arguments. - # - # Gitaly note: JV: to be deprecated in favor of Commit.between - def rugged_commits_between(from, to) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) - - sha_from = sha_from_ref(from) - sha_to = sha_from_ref(to) + def count_commits(options) + options = process_count_commits_options(options.dup) - walker.push(sha_to) - walker.hide(sha_from) + wrapped_gitaly_errors do + if options[:left_right] + from = options[:from] + to = options[:to] - commits = walker.to_a - walker.reset + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) - commits + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end + end end # Counts the amount of commits between `from` and `to`. @@ -584,65 +382,31 @@ module Gitlab def raw_changes_between(old_rev, new_rev) @raw_changes_between ||= {} - @raw_changes_between[[old_rev, new_rev]] ||= begin - return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA + @raw_changes_between[[old_rev, new_rev]] ||= + begin + return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA - gitaly_migrate(:raw_changes_between) do |is_enabled| - if is_enabled + wrapped_gitaly_errors do gitaly_repository_client.raw_changes_between(old_rev, new_rev) .each_with_object([]) do |msg, arr| msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) } end - else - result = [] - - circuit_breaker.perform do - Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads| - last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) } - - if wait_threads.any? { |waiter| !waiter.value&.success? } - raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}" - end - end - end - - result end end - end rescue ArgumentError => e raise Gitlab::Git::Repository::GitError.new(e) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base(from, to) - gitaly_migrate(:merge_base) do |is_enabled| - if is_enabled - gitaly_repository_client.find_merge_base(from, to) - else - rugged_merge_base(from, to) - end + def merge_base(*commits) + wrapped_gitaly_errors do + gitaly_repository_client.find_merge_base(*commits) end end - # Gitaly note: JV: check gitlab-ee before removing this method. - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| - if is_enabled - gitaly_commit_client.ancestor?(from, to) - else - rugged_is_ancestor?(from, to) - end - end + gitaly_commit_client.ancestor?(from, to) end def merged_branch_names(branch_names = []) @@ -652,12 +416,8 @@ module Gitlab return [] unless root_sha - branches = gitaly_migrate(:merged_branch_names) do |is_enabled| - if is_enabled - gitaly_merged_branch_names(branch_names, root_sha) - else - git_merged_branch_names(branch_names, root_sha) - end + branches = wrapped_gitaly_errors do + gitaly_merged_branch_names(branch_names, root_sha) end Set.new(branches) @@ -668,35 +428,33 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - iterator = gitaly_migrate(:diff_between) do |is_enabled| - if is_enabled - gitaly_commit_client.diff(from, to, options.merge(paths: paths)) - else - diff_patches(from, to, options, *paths) - end - end + iterator = gitaly_commit_client.diff(from, to, options.merge(paths: paths)) Gitlab::Git::DiffCollection.new(iterator, options) end + def diff_stats(left_id, right_id) + if [left_id, right_id].any? { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) } + return empty_diff_stats + end + + stats = wrapped_gitaly_errors do + gitaly_commit_client.diff_stats(left_id, right_id) + end + + Gitlab::Git::DiffStatsCollection.new(stats) + rescue CommandError, TypeError + empty_diff_stats + end + # Returns a RefName for a given SHA def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end - # Get refs hash which key is is the commit id + # Get refs hash which key is the commit id # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch # Note that both inherit from Gitlab::Git::Ref def refs_hash @@ -713,41 +471,22 @@ module Gitlab @refs_hash end - # Lookup for rugged object by oid or ref name - def lookup(oid_or_ref_name) - rugged.rev_parse(oid_or_ref_name) - end - # Returns url for submodule # # Ex. # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) - Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| - if is_enabled - gitaly_submodule_url_for(ref, path) - else - if submodules(ref).any? - submodule = submodules(ref)[path] - submodule['url'] if submodule - end - end + wrapped_gitaly_errors do + gitaly_submodule_url_for(ref, path) end end # Return total commits count accessible from passed ref - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - gitaly_migrate(:commit_count) do |is_enabled| - if is_enabled - gitaly_commit_client.commit_count(ref) - else - rugged_commit_count(ref) - end + wrapped_gitaly_errors do + gitaly_commit_client.commit_count(ref) end end @@ -776,38 +515,32 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_operation_client.user_create_branch(branch_name, user, target) - rescue GRPC::FailedPrecondition => ex - raise InvalidRef, ex + wrapped_gitaly_errors do + gitaly_operation_client.user_create_branch(branch_name, user, target) + end end def add_tag(tag_name, user:, target:, message: nil) - gitaly_migrate(:operation_user_add_tag, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_add_tag(tag_name, user: user, target: target, message: message) - else - rugged_add_tag(tag_name, user: user, target: target, message: message) - end + wrapped_gitaly_errors do + gitaly_operation_client.add_tag(tag_name, user, target, message) + end + end + + def update_branch(branch_name, user:, newrev:, oldrev:) + wrapped_gitaly_errors do + gitaly_operation_client.user_update_branch(branch_name, user, newrev, oldrev) end end def rm_branch(branch_name, user:) - gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_operations_client.user_delete_branch(branch_name, user) - else - OperationService.new(user, self).rm_branch(find_branch(branch_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_delete_branch(branch_name, user) end end def rm_tag(tag_name, user:) - gitaly_migrate(:operation_user_delete_tag) do |is_enabled| - if is_enabled - gitaly_operations_client.rm_tag(tag_name, user) - else - Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.rm_tag(tag_name, user) end end @@ -816,142 +549,73 @@ module Gitlab end def merge(user, source_sha, target_branch, message, &block) - gitaly_migrate(:operation_user_merge_branch) do |is_enabled| - if is_enabled - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) - else - rugged_merge(user, source_sha, target_branch, message, &block) - end - end - end - - def rugged_merge(user, source_sha, target_branch, message) - committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) - - OperationService.new(user, self).with_branch(target_branch) do |start_commit| - our_commit = start_commit.sha - their_commit = source_sha - - raise 'Invalid merge target' unless our_commit - raise 'Invalid merge source' unless their_commit - - merge_index = rugged.merge_commits(our_commit, their_commit) - break if merge_index.conflicts? - - options = { - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - message: message, - author: committer, - committer: committer - } - - commit_id = create_commit(options) - - yield commit_id - - commit_id + wrapped_gitaly_errors do + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) end - rescue Gitlab::Git::CommitError # when merge_index.conflicts? - nil end def ff_merge(user, source_sha, target_branch) - gitaly_migrate(:operation_user_ff_branch) do |is_enabled| - if is_enabled - gitaly_ff_merge(user, source_sha, target_branch) - else - rugged_ff_merge(user, source_sha, target_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) end end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:revert, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } - - if is_enabled - gitaly_operations_client.user_revert(args) - else - rugged_revert(args) - end - end - end - - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:cherry_pick) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } - - if is_enabled - gitaly_operations_client.user_cherry_pick(args) - else - rugged_cherry_pick(args) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_revert(args) end end - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end + def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - def user_to_committer(user) - Gitlab::Git.committer_hash(email: user.email, name: user.name) + wrapped_gitaly_errors do + gitaly_operation_client.user_cherry_pick(args) + end end - def create_commit(params = {}) - params[:message].delete!("\r") + def update_submodule(user:, submodule:, commit_sha:, message:, branch:) + args = { + user: user, + submodule: submodule, + commit_sha: commit_sha, + branch: branch, + message: message + } - Rugged::Commit.create(rugged, params) + wrapped_gitaly_errors do + gitaly_operation_client.user_update_submodule(args) + end end # Delete the specified branch from the repository def delete_branch(branch_name) - gitaly_migrate(:delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.delete_branch(branch_name) - else - rugged.branches.delete(branch_name) - end + wrapped_gitaly_errors do + gitaly_ref_client.delete_branch(branch_name) end - rescue Rugged::ReferenceError, CommandError => e + rescue CommandError => e raise DeleteBranchError, e end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_delete_refs(*ref_names) - else - git_delete_refs(*ref_names) - end + wrapped_gitaly_errors do + gitaly_delete_refs(*ref_names) end end @@ -961,58 +625,30 @@ module Gitlab # create_branch("feature") # create_branch("other-feature", "master") def create_branch(ref, start_point = "HEAD") - gitaly_migrate(:create_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_ref_client.create_branch(ref, start_point) - else - rugged_create_branch(ref, start_point) - end + wrapped_gitaly_errors do + gitaly_ref_client.create_branch(ref, start_point) end end # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - gitaly_migrate(:remote_add_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) - else - rugged_add_remote(remote_name, url, mirror_refmap) - end + wrapped_gitaly_errors do + gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) end end def remove_remote(remote_name) - gitaly_migrate(:remote_remove_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.remove_remote(remote_name) - else - rugged_remove_remote(remote_name) - end + wrapped_gitaly_errors do + gitaly_remote_client.remove_remote(remote_name) end end - # Update the specified remote using the values in the +options+ hash - # - # Example - # repo.update_remote("origin", url: "path/to/repo") - def remote_update(remote_name, url:) - # TODO: Implement other remote options - rugged.remotes.set_url(remote_name, url) - nil - end + def find_remote_root_ref(remote_name) + return unless remote_name.present? - AUTOCRLF_VALUES = { - "true" => true, - "false" => false, - "input" => :input - }.freeze - - def autocrlf - AUTOCRLF_VALUES[rugged.config['core.autocrlf']] - end - - def autocrlf=(value) - rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + wrapped_gitaly_errors do + gitaly_remote_client.find_remote_root_ref(remote_name) + end end # Returns result like "git ls-files" , recursive and full file path @@ -1020,48 +656,20 @@ module Gitlab # Ex. # repo.ls_files('master') # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) - Gitlab::GitalyClient.migrate(:apply_gitattributes) do |is_enabled| - if is_enabled - gitaly_copy_gitattributes(ref) - else - rugged_copy_gitattributes(ref) - end + wrapped_gitaly_errors do + gitaly_repository_client.apply_gitattributes(ref) end - rescue GRPC::InvalidArgument - raise InvalidRef end def info_attributes return @info_attributes if @info_attributes - content = - gitaly_migrate(:get_info_attributes, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.info_attributes - else - attributes_path = File.join(File.expand_path(path), 'info', 'attributes') - - if File.exist?(attributes_path) - File.read(attributes_path) - else - "" - end - end - end - + content = gitaly_repository_client.info_attributes @info_attributes = AttributesParser.new(content) end @@ -1090,147 +698,46 @@ module Gitlab end def languages(ref = nil) - gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.languages(ref) - else - ref ||= rugged.head.target_id - languages = Linguist::Repository.new(rugged, ref).languages - total = languages.map(&:last).sum - - languages = languages.map do |language| - name, share = language - color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" - { - value: (share.to_f * 100 / total).round(2), - label: name, - color: color, - highlight: color - } - end - - languages.sort do |x, y| - y[:value] <=> x[:value] - end - end + wrapped_gitaly_errors do + gitaly_commit_client.languages(ref) end end def license_short_name - gitaly_migrate(:license_short_name, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.license_short_name - else - begin - # The licensee gem creates a Rugged object from the path: - # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb - Licensee.license(path).try(:key) - rescue Rugged::Error - end - end + wrapped_gitaly_errors do + gitaly_repository_client.license_short_name end end - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end + def fetch_source_branch!(source_repository, source_branch, local_ref) + wrapped_gitaly_errors do + gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) end end - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name + def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + tmp_ref = "refs/tmp/#{SecureRandom.hex}" - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end + return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" + Gitlab::Git::Compare.new( + self, + target_branch_name, + tmp_ref, + straight: straight ) - - yield commit(sha) ensure - delete_refs(tmp_ref) if tmp_ref - end - - def fetch_source_branch!(source_repository, source_branch, local_ref) - Gitlab::GitalyClient.migrate(:fetch_source_branch) do |is_enabled| - if is_enabled - gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) - else - rugged_fetch_source_branch(source_repository, source_branch, local_ref) - end - end + delete_refs(tmp_ref) end - def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - with_repo_branch_commit(source_repository, source_branch_name) do |commit| - break unless commit - - Gitlab::Git::Compare.new( - self, - target_branch_name, - commit.sha, - straight: straight - ) - end - end - end - - def write_ref(ref_path, ref, old_ref: nil, shell: true) + def write_ref(ref_path, ref, old_ref: nil) ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD" - gitaly_migrate(:write_ref) do |is_enabled| - if is_enabled - gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) - else - local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell) - end + wrapped_gitaly_errors do + gitaly_repository_client.write_ref(ref_path, ref, old_ref) end end - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - message, status = GitalyClient.migrate(:fetch_ref) do |is_enabled| - if is_enabled - gitaly_fetch_ref(source_repository, source_ref: source_ref, target_ref: target_ref) - else - # When removing this code, also remove source_repository#path - # to remove deprecated method calls - local_fetch_ref(source_repository.path, source_ref: source_ref, target_ref: target_ref) - end - end - - # Make sure ref was created, and raise Rugged::ReferenceError when not - raise Rugged::ReferenceError, message if status != 0 - - target_ref - end - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) @@ -1241,12 +748,28 @@ module Gitlab end def fetch_repository_as_mirror(repository) - gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| - if is_enabled - gitaly_remote_client.fetch_internal_remote(repository) - else - rugged_fetch_repository_as_mirror(repository) - end + wrapped_gitaly_errors do + gitaly_remote_client.fetch_internal_remote(repository) + end + end + + # Fetch remote for repository + # + # remote - remote name + # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # prune - should we use --prune flag? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + wrapped_gitaly_errors do + gitaly_repository_client.fetch_remote( + remote, + ssh_auth: ssh_auth, + forced: forced, + no_tags: no_tags, + prune: prune, + timeout: GITLAB_PROJECTS_TIMEOUT + ) end end @@ -1259,20 +782,6 @@ module Gitlab Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end - def commit_index(user, branch_name, index, options) - committer = user_to_committer(user) - - OperationService.new(user, self).with_branch(branch_name) do - commit_params = options.merge( - tree: index.write_tree(rugged), - author: committer, - committer: committer - ) - - create_commit(commit_params) - end - end - def fsck msg, status = gitaly_repository_client.fsck @@ -1280,16 +789,12 @@ module Gitlab end def create_from_bundle(bundle_path) - gitaly_migrate(:create_repo_from_bundle) do |is_enabled| - if is_enabled - gitaly_repository_client.create_from_bundle(bundle_path) - else - run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) - self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) - end - end + # It's important to check that the linked-to file is actually a valid + # .bundle file as it is passed to `git clone`, which may otherwise + # interpret it as a pointer to another repository + ::Gitlab::Git::BundleFile.check!(bundle_path) - true + gitaly_repository_client.create_from_bundle(bundle_path) end def create_from_snapshot(url, auth) @@ -1297,121 +802,77 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_migrate(:rebase) do |is_enabled| - if is_enabled - gitaly_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - else - git_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) end end def rebase_in_progress?(rebase_id) - gitaly_migrate(:rebase_in_progress) do |is_enabled| - if is_enabled - gitaly_repository_client.rebase_in_progress?(rebase_id) - else - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.rebase_in_progress?(rebase_id) end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - gitaly_migrate(:squash) do |is_enabled| - if is_enabled - gitaly_operation_client.user_squash(user, squash_id, branch, + wrapped_gitaly_errors do + gitaly_operation_client.user_squash(user, squash_id, branch, start_sha, end_sha, author, message) - else - git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - end end end def squash_in_progress?(squash_id) - gitaly_migrate(:squash_in_progress, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.squash_in_progress?(squash_id) - else - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.squash_in_progress?(squash_id) end end - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - def bundle_to_disk(save_path) - gitaly_migrate(:bundle_to_disk) do |is_enabled| - if is_enabled - gitaly_repository_client.create_bundle(save_path) - else - run_git!(%W(bundle create #{save_path} --all)) - end + wrapped_gitaly_errors do + gitaly_repository_client.create_bundle(save_path) end true end - # rubocop:disable Metrics/ParameterLists def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_repository: self) - gitaly_migrate(:operation_user_commit_files) do |is_enabled| - if is_enabled - gitaly_operation_client.user_commit_files(user, branch_name, + wrapped_gitaly_errors do + gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, start_branch_name, start_repository) - else - rugged_multi_action(user, branch_name, message, actions, - author_email, author_name, start_branch_name, start_repository) - end end end - # rubocop:enable Metrics/ParameterLists def write_config(full_path:) return unless full_path.present? - gitaly_migrate(:write_config) do |is_enabled| - if is_enabled - gitaly_repository_client.write_config(full_path: full_path) - else - rugged_write_config(full_path: full_path) - end + # This guard avoids Gitaly log/error spam + raise NoRepository, 'repository does not exist' unless exists? + + set_config('gitlab.fullpath' => full_path) + end + + def set_config(entries) + wrapped_gitaly_errors do + gitaly_repository_client.set_config(entries) end end - def gitaly_repository - Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) + def delete_config(*keys) + wrapped_gitaly_errors do + gitaly_repository_client.delete_config(keys) + end end - def gitaly_operations_client - @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) + def gitaly_repository + Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end def gitaly_ref_client @@ -1442,19 +903,9 @@ module Gitlab Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) - end - def clean_stale_repository_files - gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - gitaly_repository_client.cleanup if is_enabled && exists? + wrapped_gitaly_errors do + gitaly_repository_client.cleanup if exists? end rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup Rails.logger.error("Unable to clean repository on storage #{storage} with relative path #{relative_path}: #{e.message}") @@ -1478,25 +929,14 @@ module Gitlab safe_query = Regexp.escape(query) ref ||= root_ref - gitaly_migrate(:search_files_by_content) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_content(ref, safe_query) - else - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - - run_git(args).first.scrub.split(/^--\n/) - end - end + gitaly_repository_client.search_files_by_content(ref, safe_query) end def can_be_merged?(source_sha, target_branch) - gitaly_migrate(:can_be_merged) do |is_enabled| - if is_enabled - gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) - else - rugged_can_be_merged?(source_sha, target_branch) - end + if target_sha = find_branch(target_branch)&.target + !gitaly_conflicts_client(source_sha, target_sha).conflicts? + else + false end end @@ -1506,87 +946,27 @@ module Gitlab return [] if empty? || safe_query.blank? - gitaly_migrate(:search_files_by_name) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_name(ref, safe_query) - else - args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - - run_git(args).first.lines.map(&:strip) - end - end + gitaly_repository_client.search_files_by_name(ref, safe_query) end def find_commits_by_message(query, ref, path, limit, offset) - gitaly_migrate(:commits_by_message) do |is_enabled| - if is_enabled - find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - else - find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - end + wrapped_gitaly_errors do + gitaly_commit_client + .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) + .map { |c| commit(c) } end end - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output - end - - def last_commit_for_path(sha, path) - gitaly_migrate(:last_commit_for_path) do |is_enabled| - if is_enabled - last_commit_for_path_by_gitaly(sha, path) - else - last_commit_for_path_by_rugged(sha, path) - end - end - end - - def rev_list(including: [], excluding: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25) + wrapped_gitaly_errors do + gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit) end - - args.push('--objects') if objects - - run_git!(args, lazy_block: block) - end - - def missed_ref(oldrev, newrev) - run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) - base_args = %w(worktree add --detach) - - # Note that we _don't_ want to test for `.present?` here: If the caller - # passes an non nil empty value it means it still wants sparse checkout - # but just isn't interested in any file, perhaps because it wants to - # checkout files in by a changeset but that changeset only adds files. - if sparse_checkout_files - # Create worktree without checking out - run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - - configure_sparse_checkout(worktree_git_path, sparse_checkout_files) - - # After sparse checkout configuration, checkout `branch` in worktree - run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) - else - # Create worktree and checkout `branch` in it - run_git!(base_args + [worktree_path, branch], env: env) + def last_commit_for_path(sha, path) + wrapped_gitaly_errors do + gitaly_commit_client.last_commit_for_path(sha, path) end - - yield - ensure - FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) - FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) end def checksum @@ -1600,157 +980,13 @@ module Gitlab private - def uncached_has_local_branches? - gitaly_migrate(:has_local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.has_local_branches? - else - has_local_branches_rugged? - end - end - end - - def local_write_ref(ref_path, ref, old_ref: nil, shell: true) - if shell - shell_write_ref(ref_path, ref, old_ref) - else - rugged_write_ref(ref_path, ref) - end - end - - def rugged_write_config(full_path:) - rugged.config['gitlab.fullpath'] = full_path - end - - def shell_write_ref(ref_path, ref, old_ref) - raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') - raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") - raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") - - input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" - run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } - end - - def rugged_write_ref(ref_path, ref) - rugged.references.create(ref_path, ref, force: true) - rescue Rugged::ReferenceError => ex - Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" - rescue Rugged::OSError => ex - raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/ - - Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" - end - - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def fresh_worktree?(path) - File.exist?(path) && !clean_stuck_worktree(path) + def empty_diff_stats + Gitlab::Git::DiffStatsCollection.new([]) end - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def clean_stuck_worktree(path) - return false unless File.mtime(path) < 15.minutes.ago - - FileUtils.rm_rf(path) - true - end - - # Adding a worktree means checking out the repository. For large repos, - # this can be very expensive, so set up sparse checkout for the worktree - # to only check out the files we're interested in. - def configure_sparse_checkout(worktree_git_path, files) - run_git!(%w(config core.sparseCheckout true)) - - return if files.empty? - - worktree_info_path = File.join(worktree_git_path, 'info') - FileUtils.mkdir_p(worktree_info_path) - File.write(File.join(worktree_info_path, 'sparse-checkout'), files) - end - - def rugged_fetch_source_branch(source_repository, source_branch, local_ref) - with_repo_branch_commit(source_repository, source_branch) do |commit| - if commit - write_ref(local_ref, commit.sha) - true - else - false - end - end - end - - def worktree_path(prefix, id) - id = id.to_s - raise ArgumentError, "worktree id can't be empty" unless id.present? - raise ArgumentError, "worktree id can't contain slashes " if id.include?("/") - - File.join(path, 'gitlab-worktree', "#{prefix}-#{id}") - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - - # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. - def branches_filter(filter: nil, sort_by: nil) - branches = rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - - sort_branches(branches, sort_by) - end - - def git_merged_branch_names(branch_names, root_sha) - git_arguments = - %W[branch --merged #{root_sha} - --format=%(refname:short)\ %(objectname)] + branch_names - - lines = run_git(git_arguments).first.lines - - lines.each_with_object([]) do |line, branches| - name, sha = line.strip.split(' ', 2) - - branches << name if sha != root_sha + def uncached_has_local_branches? + wrapped_gitaly_errors do + gitaly_repository_client.has_local_branches? end end @@ -1782,63 +1018,6 @@ module Gitlab end end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the - # complexity is such that we should migrate it as Ruby before trying to - # do it in Go. - def log_by_shell(sha, options) - limit = options[:limit].to_i - offset = options[:offset].to_i - use_follow_flag = options[:follow] && options[:path].present? - - # We will perform the offset in Ruby because --follow doesn't play well with --skip. - # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 - offset_in_ruby = use_follow_flag && options[:offset].present? - limit += offset if offset_in_ruby - - cmd = %w[log] - cmd << "--max-count=#{limit}" - cmd << '--format=%H' - cmd << "--skip=#{offset}" unless offset_in_ruby - cmd << '--follow' if use_follow_flag - cmd << '--no-merges' if options[:skip_merges] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - - if options[:all] - cmd += %w[--all --reverse] - else - cmd << sha - end - - # :path can be a string or an array of strings - if options[:path].present? - cmd << '--' - cmd += Array(options[:path]) - end - - raw_output, _status = run_git(cmd) - lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines - - lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } - end - - # We are trying to deprecate this method because it does a lot of work - # but it seems to be used only to look up submodule URL's. - # https://gitlab.com/gitlab-org/gitaly/issues/329 - def submodules(ref) - commit = rev_parse_target(ref) - return {} unless commit - - begin - content = blob_content(commit, ".gitmodules") - rescue InvalidBlobName - return {} - end - - parser = GitmodulesParser.new(content) - fill_submodule_ids(commit, parser.parse) - end - def gitaly_submodule_url_for(ref, path) # We don't care about the contents so 1 byte is enough. Can't request 0 bytes, 0 means unlimited. commit_object = gitaly_commit_client.tree_entry(ref, path, 1) @@ -1853,246 +1032,6 @@ module Gitlab found_module && found_module['url'] end - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - # Get the content of a blob for a given commit. If the blob is a commit - # (for submodules) then return the blob's OID. - def blob_content(commit, blob_name) - blob_entry = tree_entry(commit, blob_name) - - unless blob_entry - raise InvalidBlobName.new("Invalid blob name: #{blob_name}") - end - - case blob_entry[:type] - when :commit - blob_entry[:oid] - when :tree - raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") - when :blob - rugged.lookup(blob_entry[:oid]).content - end - end - - # Fill in the 'id' field of a submodule hash from its values - # as-of +commit+. Return a Hash consisting only of entries - # from the submodule hash for which the 'id' field is filled. - def fill_submodule_ids(commit, submodule_data) - submodule_data.each do |path, data| - id = begin - blob_content(commit, path) - rescue InvalidBlobName - nil - end - data['id'] = id - end - submodule_data.select { |path, data| data['id'] } - end - - # Find the entry for +path+ in the tree for +commit+ - def tree_entry(commit, path) - pathname = Pathname.new(path) - first = true - tmp_entry = nil - - pathname.each_filename do |dir| - if first - tmp_entry = commit.tree[dir] - first = false - elsif tmp_entry.nil? - return nil - else - begin - tmp_entry = rugged.lookup(tmp_entry[:oid]) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - return nil - end - - return nil unless tmp_entry.type == :tree - - tmp_entry = tmp_entry[dir] - end - end - - tmp_entry - end - - # Return the Rugged patches for the diff between +from+ and +to+. - def diff_patches(from, to, options = {}, *paths) - options ||= {} - break_rewrites = options[:break_rewrites] - actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) - - diff = rugged.diff(from, to, actual_options) - diff.find_similar!(break_rewrites: break_rewrites) - diff.each_patch - end - - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - - def tags_from_rugged - rugged.references.each("refs/tags/*").map do |ref| - message = nil - - if ref.target.is_a?(Rugged::Tag::Annotation) - tag_message = ref.target.message - - if tag_message.respond_to?(:chomp) - message = tag_message.chomp - end - end - - target_commit = Gitlab::Git::Commit.find(self, ref.target) - Gitlab::Git::Tag.new(self, { - name: ref.name, - target: ref.target, - target_commit: target_commit, - message: message - }) - end.sort_by(&:name) - end - - def last_commit_for_path_by_rugged(sha, path) - sha = last_commit_id_for_path_by_shelling_out(sha, path) - commit(sha) - end - - def tags_from_gitaly - gitaly_ref_client.tags - end - - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def count_commits_by_gitaly(options) - if options[:left_right] - from = options[:from] - to = options[:to] - - right_count = gitaly_commit_client - .commit_count("#{from}..#{to}", options) - left_count = gitaly_commit_client - .commit_count("#{to}..#{from}", options) - - [left_count, right_count] - else - gitaly_commit_client.commit_count(options[:ref], options) - end - end - - def count_commits_by_shelling_out(options) - cmd = count_commits_shelling_command(options) - - raw_output, _status = run_git(cmd) - - process_count_commits_raw_output(raw_output, options) - end - - def count_commits_shelling_command(options) - cmd = %w[rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << "--max-count=#{options[:max_count]}" if options[:max_count] - cmd << "--left-right" if options[:left_right] - cmd << '--count' - - cmd << if options[:all] - '--all' - elsif options[:ref] - options[:ref] - else - raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" - end - - cmd += %W[-- #{options[:path]}] if options[:path].present? - cmd - end - - def process_count_commits_raw_output(raw_output, options) - if options[:left_right] - result = raw_output.scan(/\d+/).map(&:to_i) - - if result.sum != options[:max_count] - result - else # Reaching max count, right is not accurate - right_option = - process_count_commits_options(options - .except(:left_right, :from, :to) - .merge(ref: options[:to])) - - right = count_commits_by_shelling_out(right_option) - - [result.first, right] # left should be accurate in the first call - end - else - raw_output.to_i - end - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - - # Returns true if the given ref name exists - # - # Ref names must start with `refs/`. - def rugged_ref_exists?(ref_name) - raise ArgumentError, 'invalid refname' unless ref_name.start_with?('refs/') - - rugged.references.exist?(ref_name) - rescue Rugged::ReferenceError - false - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -2100,433 +1039,13 @@ module Gitlab gitaly_ref_client.ref_exists?(ref_name) end - # Returns true if the given tag exists - # - # name - The name of the tag as a String. - def rugged_tag_exists?(name) - !!rugged.tags[name] - end - - # Returns true if the given branch exists - # - # name - The name of the branch as a String. - def rugged_branch_exists?(name) - rugged.branches.exists?(name) - - # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. - # Whatever code calls this method shouldn't have to deal with that so - # instead we just return `false` (which is true since a branch doesn't - # exist when it has an invalid name). - rescue Rugged::ReferenceError - false - end - - def gitaly_add_tag(tag_name, user:, target:, message: nil) - gitaly_operations_client.add_tag(tag_name, user, target, message) - end - - def rugged_add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } - end - - Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex - rescue Rugged::TagError - raise TagExistsError - end - - def rugged_create_branch(ref, start_point) - rugged_ref = rugged.branches.create(ref, start_point) - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'} - - raise InvalidRef.new("Invalid reference #{start_point}") - end - def gitaly_copy_gitattributes(revision) gitaly_repository_client.apply_gitattributes(revision) end - def rugged_copy_gitattributes(ref) - begin - commit = lookup(ref) - rescue Rugged::ReferenceError - raise InvalidRef.new("Ref #{ref} is invalid") - end - - # Create the paths - info_dir_path = File.join(path, 'info') - info_attributes_path = File.join(info_dir_path, 'attributes') - - begin - # Retrieve the contents of the blob - gitattributes_content = blob_content(commit, '.gitattributes') - rescue InvalidBlobName - # No .gitattributes found. Should now remove any info/attributes and return - File.delete(info_attributes_path) if File.exist?(info_attributes_path) - return - end - - # Create the info directory if needed - Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) - - # Write the contents of the .gitattributes file to info/attributes - # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 - File.open(info_attributes_path, "wb") do |file| - file.write(gitattributes_content) - end - end - - def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) - - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) - end - end - - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - cherry_pick_tree_id = check_cherry_pick_content(commit, start_commit.sha) - raise CreateTreeError unless cherry_pick_tree_id - - committer = user_to_committer(user) - - create_commit(message: message, - author: { - email: commit.author_email, - name: commit.author_name, - time: commit.authored_date - }, - committer: committer, - tree: cherry_pick_tree_id, - parents: [start_commit.sha]) - end - end - - def check_cherry_pick_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << 1 if target_commit.merge_commit? - - cherry_pick_index = rugged.cherrypick_commit(*args) - return false if cherry_pick_index.conflicts? - - tree_id = cherry_pick_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - - def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_operation_client.user_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end - - def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - if remote_repository.is_a?(RemoteRepository) - env.merge!(remote_repository.fetch_env) - remote_repo_path = GITALY_INTERNAL_URL - else - remote_repo_path = remote_repository.path - end - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repo_path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha - end - end - - def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=ar --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| - stdin.binmode - stdin.write(diff) - end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp - end - end - - def local_fetch_ref(source_path, source_ref:, target_ref:) - args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) - run_git(args) - end - - def gitaly_fetch_ref(source_repository, source_ref:, target_ref:) - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - - run_git(args, env: source_repository.fetch_env) - end - - def gitaly_ff_merge(user, source_sha, target_branch) - gitaly_operations_client.user_ff_branch(user, source_sha, target_branch) - rescue GRPC::FailedPrecondition => e - raise CommitError, e - end - - def rugged_ff_merge(user, source_sha, target_branch) - OperationService.new(user, self).with_branch(target_branch) do |our_commit| - raise ArgumentError, 'Invalid merge target' unless our_commit - - source_sha - end - rescue Rugged::ReferenceError, InvalidRef - raise ArgumentError, 'Invalid merge source' - end - - def rugged_add_remote(remote_name, url, mirror_refmap) - rugged.remotes.create(remote_name, url) - - set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap - rescue Rugged::ConfigError - remote_update(remote_name, url: url) - end - - def git_delete_refs(*ref_names) - instructions = ref_names.map do |ref| - "delete #{ref}\x00\x00" - end - - message, status = run_git(%w[update-ref --stdin -z]) do |stdin| - stdin.write(instructions.join) - end - - unless status.zero? - raise GitError.new("Could not delete refs #{ref_names}: #{message}") - end - end - def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - - def rugged_remove_remote(remote_name) - # When a remote is deleted all its remote refs are deleted too, but in - # the case of mirrors we map its refs (that would usualy go under - # [remote_name]/) to the top level namespace. We clean the mapping so - # those don't get deleted. - if rugged.config["remote.#{remote_name}.mirror"] - rugged.config.delete("remote.#{remote_name}.fetch") - end - - rugged.remotes.delete(remote_name) - true - rescue Rugged::ConfigError - false - end - - def rugged_fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - - add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: repository.fetch_env) - ensure - remove_remote(remote_name) - end - - def rugged_multi_action( - user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository) - - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - index = Gitlab::Git::Index.new(self) - parents = [] - - if start_commit - index.read_tree(start_commit.rugged_commit.tree) - parents = [start_commit.sha] - end - - actions.each { |opts| index.apply(opts.delete(:action), opts) } - - committer = user_to_committer(user) - author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer - options = { - tree: index.write_tree, - message: message, - parents: parents, - author: author, - committer: committer - } - - create_commit(options) - end - end - - def fetch_remote(remote_name = 'origin', env: nil) - run_git(['fetch', remote_name], env: env).last.zero? - end - - def gitaly_can_be_merged?(their_commit, our_commit) - !gitaly_conflicts_client(our_commit, their_commit).conflicts? - end - - def rugged_can_be_merged?(their_commit, our_commit) - !rugged.merge_commits(our_commit, their_commit).conflicts? - end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset) - ref ||= root_ref - - args = %W( - log #{ref} --pretty=%H --skip #{offset} - --max-count #{limit} --grep=#{query} --regexp-ignore-case - ) - args = args.concat(%W(-- #{path})) if path.present? - - git_log_results = run_git(args).first.lines - - git_log_results.map { |c| commit(c.chomp) }.compact - end - - def find_commits_by_message_by_gitaly(query, ref, path, limit, offset) - gitaly_commit_client - .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) - .map { |c| commit(c) } - end - - def last_commit_for_path_by_gitaly(sha, path) - gitaly_commit_client.last_commit_for_path(sha, path) - end - - def last_commit_id_for_path_by_shelling_out(sha, path) - args = %W(rev-list --max-count=1 #{sha} -- #{path}) - run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip - end - - def rugged_merge_base(from, to) - rugged.merge_base(from, to) - rescue Rugged::ReferenceError - nil - end - - def rugged_commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count - rescue Rugged::ReferenceError - 0 - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end - - def sha_from_ref(ref) - rev_parse_target(ref).oid - end - - def build_git_cmd(*args) - object_directories = alternate_object_directories.join(File::PATH_SEPARATOR) - - env = { 'PWD' => self.path } - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories if object_directories.present? - - [ - env, - ::Gitlab.config.git.bin_path, - *args, - { chdir: self.path } - ] - end - - def git_diff_cmd(old_rev, new_rev) - old_rev = old_rev == ::Gitlab::Git::BLANK_SHA ? ::Gitlab::Git::EMPTY_TREE_ID : old_rev - - build_git_cmd('diff', old_rev, new_rev, '--raw') - end - - def git_cat_file_cmd - format = '%(objectname) %(objectsize) %(rest)' - build_git_cmd('cat-file', "--batch-check=#{format}") - end - - def format_git_cat_file_script - File.expand_path('../support/format-git-cat-file-input', __FILE__) - end end end end diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb new file mode 100644 index 00000000000..2d1d8435cf3 --- /dev/null +++ b/lib/gitlab/git/repository_cleaner.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class RepositoryCleaner + include Gitlab::Git::WrapsGitalyErrors + + attr_reader :repository + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + end + + def apply_bfg_object_map(io) + wrapped_gitaly_errors do + gitaly_cleanup_client.apply_bfg_object_map(io) + end + end + + private + + def gitaly_cleanup_client + @gitaly_cleanup_client ||= Gitlab::GitalyClient::CleanupService.new(repository) + end + end + end +end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index e35ea5762eb..7e63a6dc7cb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -1,94 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Git module RepositoryMirroring - REFMAPS = { - # With `:all_refs`, the repository is equivalent to the result of `git clone --mirror` - all_refs: '+refs/*:refs/*', - heads: '+refs/heads/*:refs/heads/*', - tags: '+refs/tags/*:refs/tags/*' - }.freeze - - RemoteError = Class.new(StandardError) - - def set_remote_as_mirror(remote_name, refmap: :all_refs) - set_remote_refmap(remote_name, refmap) - - rugged.config["remote.#{remote_name}.mirror"] = true - rugged.config["remote.#{remote_name}.prune"] = true - end - - def remote_tags(remote) - # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n" - # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...] - list_remote_tags(remote).map do |line| - target, path = line.strip.split("\t") - - # When the remote repo does not have tags. - if target.nil? || path.nil? - Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}" - break [] - end - - name = path.split('/', 3).last - # We're only interested in tag references - # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name - next if name =~ /\^\{\}\Z/ - - target_commit = Gitlab::Git::Commit.find(self, target) - Gitlab::Git::Tag.new(self, { - name: name, - target: target, - target_commit: target_commit - }) - end.compact - end - def remote_branches(remote_name) - branches = [] - - rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| - name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') - - begin - target_commit = Gitlab::Git::Commit.find(self, ref.target) - branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end - - branches - end - - private - - def set_remote_refmap(remote_name, refmap) - Array(refmap).each_with_index do |refspec, i| - refspec = REFMAPS[refspec] || refspec - - # We need multiple `fetch` entries, but Rugged only allows replacing a config, not adding to it. - # To make sure we start from scratch, we set the first using rugged, and use `git` for any others - if i == 0 - rugged.config["remote.#{remote_name}.fetch"] = refspec - else - run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}]) - end - end - end - - def list_remote_tags(remote) - tag_list, exit_code, error = nil - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-remote --tags #{remote}) - - Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr| - tag_list = stdout.read - error = stderr.read - exit_code = wait_thr.value.exitstatus - end - - raise RemoteError, error unless exit_code.zero? - - tag_list.split("\n") + gitaly_ref_client.remote_branches(remote_name) end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb deleted file mode 100644 index 38c3a55f96f..00000000000 --- a/lib/gitlab/git/rev_list.rb +++ /dev/null @@ -1,78 +0,0 @@ -# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. - -module Gitlab - module Git - class RevList - include Gitlab::Git::Popen - - attr_reader :oldrev, :newrev, :repository - - def initialize(repository, newrev:, oldrev: nil) - @oldrev = oldrev - @newrev = newrev - @repository = repository - end - - # This method returns an array of new commit references - def new_refs - repository.rev_list(including: newrev, excluding: :all).split("\n") - end - - # Finds newly added objects - # Returns an array of shas - # - # Can skip objects which do not have a path using required_path: true - # This skips commit objects and root trees, which might not be needed when - # looking for blobs - # - # When given a block it will yield objects as a lazy enumerator so - # the caller can limit work done instead of processing megabytes of data - def new_objects(require_path: nil, not_in: nil, &lazy_block) - opts = { - including: newrev, - excluding: not_in.nil? ? :all : not_in, - require_path: require_path - } - - get_objects(opts, &lazy_block) - end - - def all_objects(require_path: nil, &lazy_block) - get_objects(including: :all, require_path: require_path, &lazy_block) - end - - # This methods returns an array of missed references - # - # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. - def missed_ref - repository.missed_ref(oldrev, newrev).split("\n") - end - - private - - def execute(args) - repository.rev_list(args).split("\n") - end - - def get_objects(including: [], excluding: [], require_path: nil) - opts = { including: including, excluding: excluding, objects: true } - - repository.rev_list(opts) do |lazy_output| - objects = objects_from_output(lazy_output, require_path: require_path) - - yield(objects) - end - end - - def objects_from_output(object_output, require_path: nil) - object_output.map do |output_line| - sha, path = output_line.split(' ', 2) - - next if require_path && path.to_s.empty? - - sha - end.reject(&:nil?) - end - end - end -end diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb deleted file mode 100644 index 5933312b0b5..00000000000 --- a/lib/gitlab/git/storage.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - module Git - module Storage - class Inaccessible < StandardError - attr_reader :retry_after - - def initialize(message = nil, retry_after = nil) - super(message) - @retry_after = retry_after - end - end - - CircuitOpen = Class.new(Inaccessible) - Misconfiguration = Class.new(Inaccessible) - Failing = Class.new(Inaccessible) - - REDIS_KEY_PREFIX = 'storage_accessible:'.freeze - REDIS_KNOWN_KEYS = "#{REDIS_KEY_PREFIX}known_keys_set".freeze - - def self.redis - Gitlab::Redis::SharedState - end - end - end -end diff --git a/lib/gitlab/git/storage/checker.rb b/lib/gitlab/git/storage/checker.rb deleted file mode 100644 index 391f0d70583..00000000000 --- a/lib/gitlab/git/storage/checker.rb +++ /dev/null @@ -1,120 +0,0 @@ -module Gitlab - module Git - module Storage - class Checker - include CircuitBreakerSettings - - attr_reader :storage_path, :storage, :hostname, :logger - METRICS_MUTEX = Mutex.new - STORAGE_TIMING_BUCKETS = [0.1, 0.15, 0.25, 0.33, 0.5, 1, 1.5, 2.5, 5, 10, 15].freeze - - def self.check_all(logger = Rails.logger) - threads = Gitlab.config.repositories.storages.keys.map do |storage_name| - Thread.new do - Thread.current[:result] = new(storage_name, logger).check_with_lease - end - end - - threads.map do |thread| - thread.join - thread[:result] - end - end - - def self.check_histogram - @check_histogram ||= - METRICS_MUTEX.synchronize do - @check_histogram || Gitlab::Metrics.histogram(:circuitbreaker_storage_check_duration_seconds, - 'Storage check time in seconds', - {}, - STORAGE_TIMING_BUCKETS - ) - end - end - - def initialize(storage, logger = Rails.logger) - @storage = storage - config = Gitlab.config.repositories.storages[@storage] - @storage_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { config.legacy_disk_path } - @logger = logger - - @hostname = Gitlab::Environment.hostname - end - - def check_with_lease - lease_key = "storage_check:#{cache_key}" - lease = Gitlab::ExclusiveLease.new(lease_key, timeout: storage_timeout) - result = { storage: storage, success: nil } - - if uuid = lease.try_obtain - result[:success] = check - - Gitlab::ExclusiveLease.cancel(lease_key, uuid) - else - logger.warn("#{hostname}: #{storage}: Skipping check, previous check still running") - end - - result - end - - def check - if perform_access_check - track_storage_accessible - true - else - track_storage_inaccessible - logger.error("#{hostname}: #{storage}: Not accessible.") - false - end - end - - private - - def perform_access_check - start_time = Gitlab::Metrics::System.monotonic_time - - Gitlab::Git::Storage::ForkedStorageCheck.storage_available?(storage_path, storage_timeout, access_retries) - ensure - execution_time = Gitlab::Metrics::System.monotonic_time - start_time - self.class.check_histogram.observe({ storage: storage }, execution_time) - end - - def track_storage_inaccessible - first_failure = current_failure_info.first_failure || Time.now - last_failure = Time.now - - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, first_failure.to_i) - redis.hset(cache_key, :last_failure, last_failure.to_i) - redis.hincrby(cache_key, :failure_count, 1) - redis.expire(cache_key, failure_reset_time) - maintain_known_keys(redis) - end - end - end - - def track_storage_accessible - Gitlab::Git::Storage.redis.with do |redis| - redis.pipelined do - redis.hset(cache_key, :first_failure, nil) - redis.hset(cache_key, :last_failure, nil) - redis.hset(cache_key, :failure_count, 0) - maintain_known_keys(redis) - end - end - end - - def maintain_known_keys(redis) - expire_time = Time.now.to_i + failure_reset_time - redis.zadd(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, expire_time, cache_key) - redis.zremrangebyscore(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, '-inf', Time.now.to_i) - end - - def current_failure_info - FailureInfo.load(cache_key) - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb deleted file mode 100644 index 62427ac9cc4..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Gitlab - module Git - module Storage - class CircuitBreaker - include CircuitBreakerSettings - - attr_reader :storage, - :hostname - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def self.for_storage(storage) - cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do - Hash.new do |hash, storage_name| - hash[storage_name] = build(storage_name) - end - end - - cached_circuitbreakers[storage] - end - - def self.build(storage, hostname = Gitlab::Environment.hostname) - config = Gitlab.config.repositories.storages[storage] - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - if !config.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Storage '#{storage}' is not configured")) - elsif !config.legacy_disk_path.present? - NullCircuitBreaker.new(storage, hostname, error: Misconfiguration.new("Path for storage '#{storage}' is not configured")) - else - new(storage, hostname) - end - end - end - - def initialize(storage, hostname) - @storage = storage - @hostname = hostname - end - - def perform - return yield unless enabled? - - check_storage_accessible! - - yield - end - - def circuit_broken? - return false if no_failures? - - failure_count > failure_count_threshold - end - - private - - # The circuitbreaker can be enabled for the entire fleet using a Feature - # flag. - # - # Enabling it for a single host can be done setting the - # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable. - def enabled? - ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker') - end - - def failure_info - @failure_info ||= FailureInfo.load(cache_key) - end - - def check_storage_accessible! - if circuit_broken? - raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time) - end - end - end - end - end -end diff --git a/lib/gitlab/git/storage/circuit_breaker_settings.rb b/lib/gitlab/git/storage/circuit_breaker_settings.rb deleted file mode 100644 index c9e225f187d..00000000000 --- a/lib/gitlab/git/storage/circuit_breaker_settings.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Gitlab - module Git - module Storage - module CircuitBreakerSettings - def failure_count_threshold - application_settings.circuitbreaker_failure_count_threshold - end - - def failure_reset_time - application_settings.circuitbreaker_failure_reset_time - end - - def storage_timeout - application_settings.circuitbreaker_storage_timeout - end - - def access_retries - application_settings.circuitbreaker_access_retries - end - - def check_interval - application_settings.circuitbreaker_check_interval - end - - def cache_key - @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" - end - - private - - def application_settings - Gitlab::CurrentSettings.current_application_settings - end - end - end - end -end diff --git a/lib/gitlab/git/storage/failure_info.rb b/lib/gitlab/git/storage/failure_info.rb deleted file mode 100644 index 387279c110d..00000000000 --- a/lib/gitlab/git/storage/failure_info.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module Git - module Storage - class FailureInfo - attr_accessor :first_failure, :last_failure, :failure_count - - def self.reset_all! - Gitlab::Git::Storage.redis.with do |redis| - all_storage_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - redis.del(*all_storage_keys) unless all_storage_keys.empty? - end - - RequestStore.delete(:circuitbreaker_cache) - end - - def self.load(cache_key) - first_failure, last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| - redis.hmget(cache_key, :first_failure, :last_failure, :failure_count) - end - - last_failure = Time.at(last_failure.to_i) if last_failure.present? - first_failure = Time.at(first_failure.to_i) if first_failure.present? - - new(first_failure, last_failure, failure_count.to_i) - end - - def initialize(first_failure, last_failure, failure_count) - @first_failure = first_failure - @last_failure = last_failure - @failure_count = failure_count - end - - def no_failures? - first_failure.blank? && last_failure.blank? && failure_count == 0 - end - end - end - end -end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb deleted file mode 100644 index 0a4e557b59b..00000000000 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ /dev/null @@ -1,65 +0,0 @@ -module Gitlab - module Git - module Storage - module ForkedStorageCheck - extend self - - def storage_available?(path, timeout_seconds = 5, retries = 1) - partial_timeout = timeout_seconds / retries - status = timeout_check(path, partial_timeout) - - # If the status check did not succeed the first time, we retry a few - # more times to avoid one-off failures - current_attempts = 1 - while current_attempts < retries && !status.success? - status = timeout_check(path, partial_timeout) - current_attempts += 1 - end - - status.success? - end - - def timeout_check(path, timeout_seconds) - filesystem_check_pid = check_filesystem_in_process(path) - - deadline = timeout_seconds.seconds.from_now.utc - wait_time = 0.01 - status = nil - - while status.nil? - - if deadline > Time.now.utc - sleep(wait_time) - _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) - else - Process.kill('KILL', filesystem_check_pid) - # Blocking wait, so we are sure the process is gone before continuing - _pid, status = Process.wait2(filesystem_check_pid) - end - end - - status - end - - # This will spawn a new 2 processes to do the check: - # The outer child (waiter) will spawn another child process (stater). - # - # The stater is the process is performing the actual filesystem check - # the check might hang if the filesystem is acting up. - # In this case we will send a `KILL` to the waiter, which will still - # be responsive while the stater is hanging. - def check_filesystem_in_process(path) - spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null') - end - - def ruby_check - <<~RUBY_FILESYSTEM_CHECK - inner_pid = fork { File.stat(ARGV.first) } - Process.waitpid(inner_pid) - exit $?.exitstatus - RUBY_FILESYSTEM_CHECK - end - end - end - end -end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb deleted file mode 100644 index 90bbe85fd37..00000000000 --- a/lib/gitlab/git/storage/health.rb +++ /dev/null @@ -1,90 +0,0 @@ -module Gitlab - module Git - module Storage - class Health - attr_reader :storage_name, :info - - def self.prefix_for_storage(storage_name) - "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:" - end - - def self.for_all_storages - storage_names = Gitlab.config.repositories.storages.keys - results_per_storage = nil - - Gitlab::Git::Storage.redis.with do |redis| - keys_per_storage = all_keys_for_storages(storage_names, redis) - results_per_storage = load_for_keys(keys_per_storage, redis) - end - - results_per_storage.map do |name, info| - info.each { |i| i[:failure_count] = i[:failure_count].value.to_i } - new(name, info) - end - end - - private_class_method def self.all_keys_for_storages(storage_names, redis) - keys_per_storage = {} - all_keys = redis.zrange(Gitlab::Git::Storage::REDIS_KNOWN_KEYS, 0, -1) - - storage_names.each do |storage_name| - prefix = prefix_for_storage(storage_name) - - keys_per_storage[storage_name] = all_keys.select { |key| key.starts_with?(prefix) } - end - - keys_per_storage - end - - private_class_method def self.load_for_keys(keys_per_storage, redis) - info_for_keys = {} - - redis.pipelined do - keys_per_storage.each do |storage_name, keys_future| - info_for_storage = keys_future.map do |key| - { name: key, failure_count: redis.hget(key, :failure_count) } - end - - info_for_keys[storage_name] = info_for_storage - end - end - - info_for_keys - end - - def self.for_failing_storages - for_all_storages.select(&:failing?) - end - - def initialize(storage_name, info) - @storage_name = storage_name - @info = info - end - - def failing_info - @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 } - end - - def failing? - failing_info.any? - end - - def failing_on_hosts - @failing_on_hosts ||= failing_info.map do |info_for_host| - info_for_host[:name].split(':').last - end - end - - def failing_circuit_breakers - @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| - CircuitBreaker.build(storage_name, hostname) - end - end - - def total_failures - @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } - end - end - end - end -end diff --git a/lib/gitlab/git/storage/null_circuit_breaker.rb b/lib/gitlab/git/storage/null_circuit_breaker.rb deleted file mode 100644 index 261c936c689..00000000000 --- a/lib/gitlab/git/storage/null_circuit_breaker.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Git - module Storage - class NullCircuitBreaker - include CircuitBreakerSettings - - # These will have actual values - attr_reader :storage, - :hostname - - # These will always have nil values - attr_reader :storage_path - - delegate :last_failure, :failure_count, :no_failures?, - to: :failure_info - - def initialize(storage, hostname, error: nil) - @storage = storage - @hostname = hostname - @error = error - end - - def perform - @error ? raise(@error) : yield - end - - def circuit_broken? - !!@error - end - - def backing_off? - false - end - - def failure_info - @failure_info ||= - if circuit_broken? - Gitlab::Git::Storage::FailureInfo.new(Time.now, - Time.now, - failure_count_threshold) - else - Gitlab::Git::Storage::FailureInfo.new(nil, - nil, - 0) - end - end - end - end - end -end diff --git a/lib/gitlab/git/support/format-git-cat-file-input b/lib/gitlab/git/support/format-git-cat-file-input deleted file mode 100755 index 2e93c646d0f..00000000000 --- a/lib/gitlab/git/support/format-git-cat-file-input +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env ruby - -# This script formats the output of the `git diff <old_rev> <new_rev> --raw` -# command so it can be processed by `git cat-file` - -# We need to convert this: -# ":100644 100644 5f53439... 85bc2f9... R060\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" -# To: -# "85bc2f9 R\tfiles/js/commit.js.coffee\tfiles/js/commit.coffee" - -ARGF.each do |line| - _, _, old_blob_id, new_blob_id, rest = line.split(/\s/, 5) - - old_blob_id.gsub!(/[^\h]/, '') - new_blob_id.gsub!(/[^\h]/, '') - - # We can't pass '0000000...' to `git cat-file` given it will not return info about the deleted file - blob_id = new_blob_id =~ /\A0+\z/ ? old_blob_id : new_blob_id - - $stdout.puts "#{blob_id} #{rest}" -end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index e44284572fd..23d989ff258 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Git class Tag < Ref @@ -12,34 +14,15 @@ module Gitlab class << self def get_message(repository, tag_id) - BatchLoader.for({ repository: repository, tag_id: tag_id }).batch do |items, loader| - items_by_repo = items.group_by { |i| i[:repository] } - - items_by_repo.each do |repo, items| - tag_ids = items.map { |i| i[:tag_id] } - - messages = get_messages(repository, tag_ids) - - messages.each do |id, message| - loader.call({ repository: repository, tag_id: id }, message) - end + BatchLoader.for(tag_id).batch(key: repository) do |tag_ids, loader, args| + get_messages(args[:key], tag_ids).each do |tag_id, message| + loader.call(tag_id, message) end end end def get_messages(repository, tag_ids) - repository.gitaly_migrate(:tag_messages) do |is_enabled| - if is_enabled - repository.gitaly_ref_client.get_tag_messages(tag_ids) - else - tag_ids.map do |id| - tag = repository.rugged.lookup(id) - message = tag.is_a?(Rugged::Commit) ? "" : tag.message - - [id, message] - end.to_h - end - end + repository.gitaly_ref_client.get_tag_messages(tag_ids) end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b6ceb542dd1..51542bcaaa2 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,9 +1,10 @@ -# Gitaly note: JV: needs 1 RPC, migration is in progress. +# frozen_string_literal: true module Gitlab module Git class Tree include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url @@ -17,12 +18,8 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' - Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) - else - tree_entries_from_rugged(repository, sha, path, recursive) - end + wrapped_gitaly_errors do + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end end @@ -56,51 +53,6 @@ module Gitlab entry[:oid] end end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end end def initialize(options) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index e573cd0e143..2c798844798 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + module Gitlab module Git class User attr_reader :username, :name, :email, :gl_id def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user)) end def self.from_gitaly(gitaly_user) diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb index 4708f22dcb3..03c2c1367b0 100644 --- a/lib/gitlab/git/util.rb +++ b/lib/gitlab/git/util.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: no RPC's here. module Gitlab diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb new file mode 100644 index 00000000000..64c89656167 --- /dev/null +++ b/lib/gitlab/git/version.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Version + def self.git_version + Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) + end + end + end +end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 1ab8c4e0229..c43331bed60 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -1,15 +1,57 @@ +# frozen_string_literal: true + module Gitlab module Git class Wiki + include Gitlab::Git::WrapsGitalyErrors + DuplicatePageError = Class.new(StandardError) OperationError = Class.new(StandardError) + DEFAULT_PAGINATION = Kaminari.config.default_per_page + CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do def to_h { user_id: user_id, username: username, name: name, email: email, message: message } end end - PageBlob = Struct.new(:name) + + # GollumSlug inlines just enough knowledge from Gollum::Page to generate a + # slug, which is used when previewing pages that haven't been persisted + class GollumSlug + class << self + def cname(name, char_white_sub = '-', char_other_sub = '-') + if name.respond_to?(:gsub) + name.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub) + else + '' + end + end + + def format_to_ext(format) + format == :markdown ? "md" : format.to_s + end + + def canonicalize_filename(filename) + ::File.basename(filename, ::File.extname(filename)).tr('-', ' ') + end + + def generate(title, format) + ext = format_to_ext(format.to_sym) + name = cname(title) + '.' + ext + canonical_name = canonicalize_filename(name) + + path = + if name.include?('/') + name.sub(%r{/[^/]+$}, '/') + else + '' + end + + path + cname(canonical_name, '-', '-') + end + end + end attr_reader :repository @@ -27,63 +69,38 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| - if is_enabled - gitaly_write_page(name, format, content, commit_details) - else - gollum_write_page(name, format, content, commit_details) - end + wrapped_gitaly_errors do + gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| - if is_enabled - gitaly_delete_page(page_path, commit_details) - else - gollum_delete_page(page_path, commit_details) - end + wrapped_gitaly_errors do + gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| - if is_enabled - gitaly_update_page(page_path, title, format, content, commit_details) - else - gollum_update_page(page_path, title, format, content, commit_details) - end + wrapped_gitaly_errors do + gitaly_update_page(page_path, title, format, content, commit_details) end end - def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| - if is_enabled - gitaly_get_all_pages - else - gollum_get_all_pages(limit: limit) - end + def pages(limit: 0) + wrapped_gitaly_errors do + gitaly_get_all_pages(limit: limit) end end def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_find_page(title: title, version: version, dir: dir) - else - gollum_find_page(title: title, version: version, dir: dir) - end + wrapped_gitaly_errors do + gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.gitaly_migrate(:wiki_find_file) do |is_enabled| - if is_enabled - gitaly_find_file(name, version) - else - gollum_find_file(name, version) - end + wrapped_gitaly_errors do + gitaly_find_file(name, version) end end @@ -92,24 +109,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| - if is_enabled - versions = gitaly_wiki_client.page_versions(page_path, options) - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page - versions[0..slice_bound] - else - current_page = gollum_page_by_path(page_path) - - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) - end - end + versions = wrapped_gitaly_errors do + gitaly_wiki_client.page_versions(page_path, options) end + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION + versions[0..slice_bound] end def count_page_versions(page_path) @@ -117,147 +125,23 @@ module Gitlab end def preview_slug(title, format) - # Adapted from gollum gem (Gollum::Wiki#preview_page) to avoid - # using Rugged through a Gollum::Wiki instance - page_class = Gollum::Page - page = page_class.new(nil) - ext = page_class.format_to_ext(format.to_sym) - name = page_class.cname(title) + '.' + ext - blob = PageBlob.new(name) - page.populate(blob) - page.url_path + GollumSlug.generate(title, format) end def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.gitaly_migrate(:wiki_page_formatted_data, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) - else - # We don't use #page because if wiki_find_page feature is enabled, we would - # get a page without formatted_data. - gollum_find_page(title: title, dir: dir, version: version)&.formatted_data - end + wrapped_gitaly_errors do + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end - def gollum_wiki - @gollum_wiki ||= Gollum::Wiki.new(@repository.path) - end - private - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def commits_from_page(gollum_page, options = {}) - unless options[:limit] - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - end - - @repository.log(ref: gollum_page.last_version.id, - path: gollum_page.path, - limit: options[:limit], - offset: options[:offset]) - end - - def gollum_page_by_path(page_path) - page_name = Gollum::Page.canonicalize_filename(page_path) - page_dir = File.split(page_path).first - - gollum_wiki.paged(page_name, page_dir) - end - - def new_page(gollum_page) - Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) - end - - def new_version(gollum_page, commit_id) - Gitlab::Git::WikiPageVersion.new(version(commit_id), gollum_page&.format) - end - - def version(commit_id) - commit_find_proc = -> { Gitlab::Git::Commit.find(@repository, commit_id) } - - if RequestStore.active? - RequestStore.fetch([:wiki_version_commit, commit_id]) { commit_find_proc.call } - else - commit_find_proc.call - end - end - - def assert_type!(object, klass) - unless object.is_a?(klass) - raise ArgumentError, "expected a #{klass}, got #{object.inspect}" - end - end - def gitaly_wiki_client @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gollum_write_page(name, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - filename = File.basename(name) - dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - - gollum_wiki.write_page(filename, format, content, { committer: committer }, dir) - end - rescue Gollum::DuplicatePageError => e - raise Gitlab::Git::Wiki::DuplicatePageError, e.message - end - - def gollum_delete_page(page_path, commit_details) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - gollum_wiki.delete_page(gollum_page_by_path(page_path), committer: committer) - end - end - - def gollum_update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - page = gollum_page_by_path(page_path) - # Instead of performing two renames if the title has changed, - # the update_page will only update the format and content and - # the rename_page will do anything related to moving/renaming - gollum_wiki.update_page(page, page.name, format, content, committer: committer) - gollum_wiki.rename_page(page, title, committer: committer) - end - end - - def gollum_find_page(title:, version: nil, dir: nil) - if version - version = Gitlab::Git::Commit.find(@repository, version).id - end - - gollum_page = gollum_wiki.page(title, version, dir) - return unless gollum_page - - new_page(gollum_page) - end - - def gollum_find_file(name, version) - version ||= self.class.default_ref - gollum_file = gollum_wiki.file(name, version) - return unless gollum_file - - Gitlab::Git::WikiFile.new(gollum_file) - end - - def gollum_get_all_pages(limit: nil) - gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } - end - def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end @@ -284,25 +168,11 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end - def gitaly_get_all_pages - gitaly_wiki_client.get_all_pages.map do |wiki_page, version| + def gitaly_get_all_pages(limit: 0) + gitaly_wiki_client.get_all_pages(limit: limit).map do |wiki_page, version| Gitlab::Git::WikiPage.new(wiki_page, version) end end - - def committer_with_hooks(commit_details) - Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) - end - - def with_committer_with_hooks(commit_details, &block) - committer = committer_with_hooks(commit_details) - - yield committer - - committer.commit - - nil - end end end end diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index 84335aca4bc..c05a5adc00c 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -1,19 +1,16 @@ +# frozen_string_literal: true + module Gitlab module Git class WikiFile attr_reader :mime_type, :raw_data, :name, :path - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::File is not serializable we must get all the data from - # 'gollum_file' during initialization, and NOT store it in an instance - # variable. - def initialize(gollum_file) - @mime_type = gollum_file.mime_type - @raw_data = gollum_file.raw_data - @name = gollum_file.name - @path = gollum_file.path + # This class wraps Gitlab::GitalyClient::WikiFile + def initialize(gitaly_file) + @mime_type = gitaly_file.mime_type + @raw_data = gitaly_file.raw_data + @name = gitaly_file.name + @path = gitaly_file.path end end end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index 669ae11a423..f6cac398548 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -1,27 +1,19 @@ +# frozen_string_literal: true + module Gitlab module Git class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Because Gollum::Page is not serializable we must get all the data from - # 'gollum_page' during initialization, and NOT store it in an instance - # variable. - # - # Note that 'version' is a WikiPageVersion instance which it itself - # serializable. That means it's OK to store 'version' in an instance - # variable. - def initialize(gollum_page, version) - @url_path = gollum_page.url_path - @title = gollum_page.title - @format = gollum_page.format - @path = gollum_page.path - @raw_data = gollum_page.raw_data - @name = gollum_page.name - @historical = gollum_page.historical? - @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page) + # This class abstracts away Gitlab::GitalyClient::WikiPage + def initialize(gitaly_page, version) + @url_path = gitaly_page.url_path + @title = gitaly_page.title + @format = gitaly_page.format + @path = gitaly_page.path + @raw_data = gitaly_page.raw_data + @name = gitaly_page.name + @historical = gitaly_page.historical? @version = version end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb index 55f1afedcab..475a9d4d1b9 100644 --- a/lib/gitlab/git/wiki_page_version.rb +++ b/lib/gitlab/git/wiki_page_version.rb @@ -1,13 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Git class WikiPageVersion attr_reader :commit, :format - # This class is meant to be serializable so that it can be constructed - # by Gitaly and sent over the network to GitLab. - # - # Both 'commit' (a Gitlab::Git::Commit) and 'format' (a string) are - # serializable. def initialize(commit, format) @commit = commit @format = format diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb new file mode 100644 index 00000000000..9963bcfbf1c --- /dev/null +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module WrapsGitalyErrors + def wrapped_gitaly_errors(&block) + yield block + rescue GRPC::NotFound => e + raise Gitlab::Git::Repository::NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) + rescue GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index db7c29be94b..010bd0e520c 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -1,12 +1,21 @@ +# frozen_string_literal: true + # Check a user's access to perform a git action. All public methods in this # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) + TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) + # Use the magic string '_any' to indicate we do not know what the + # changes are. This is also what gitlab-shell does. + ANY = '_any' + ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', @@ -19,14 +28,22 @@ module Gitlab upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', read_only: 'The repository is temporarily read-only. Please try again later.', - cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." + cannot_push_to_read_only: "You can't push code to a read-only GitLab instance.", + push_code: 'You are not allowed to push code to this project.' }.freeze - DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze - PUSH_COMMANDS = %w{ git-receive-pack }.freeze + INTERNAL_TIMEOUT = 50.seconds.freeze + LOG_HEADER = <<~MESSAGE + Push operation timed out + + Timing information for debugging purposes: + MESSAGE + + DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze + PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @@ -40,12 +57,19 @@ module Gitlab end def check(cmd, changes) + @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) + @changes = changes + check_protocol! check_valid_actor! check_active_user! check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) + + custom_action = check_custom_action(cmd) + return custom_action if custom_action + check_db_accessibility!(cmd) ensure_project_on_push!(cmd, changes) @@ -58,10 +82,10 @@ module Gitlab when *DOWNLOAD_COMMANDS check_download_access! when *PUSH_COMMANDS - check_push_access!(changes) + check_push_access! end - true + ::Gitlab::GitAccessResult::Success.new end def guest_can_download_code? @@ -88,6 +112,10 @@ module Gitlab private + def check_custom_action(cmd) + nil + end + def check_valid_actor! return unless actor.is_a?(Key) @@ -176,7 +204,7 @@ module Gitlab def ensure_project_on_push!(cmd, changes) return if project || deploy_key? - return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) namespace = Namespace.find_by_full_path(namespace_path) @@ -218,7 +246,7 @@ module Gitlab end end - def check_push_access!(changes) + def check_push_access! if project.repository_read_only? raise UnauthorizedError, ERROR_MESSAGES[:read_only] end @@ -233,38 +261,50 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access this is needed for EE. - - check_change_access!(changes) + check_change_access! end - def check_change_access!(changes) - # If there are worktrees with a HEAD pointing to a non-existent object, - # calls to `git rev-list --all` will fail in git 2.15+. This should also - # clear stale lock files. - project.repository.clean_stale_repository_files - - changes_list = Gitlab::ChangesList.new(changes) + def check_change_access! + # Deploy keys with write access can push anything + return if deploy_key? - # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each.with_index do |change, index| - first_change = index == 0 + if changes == ANY + can_push = user_access.can_do_action?(:push_code) || + project.any_branch_allows_collaboration?(user_access.user) - # If user does not have access to make at least one change, cancel all - # push by allowing the exception to bubble up - check_single_change_access(change, skip_lfs_integrity_check: !first_change) + unless can_push + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end + else + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + + # Iterate over all changes to find if user allowed all of them to be applied + changes_list.each.with_index do |change, index| + first_change = index == 0 + + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change, skip_lfs_integrity_check: !first_change) + end end end def check_single_change_access(change, skip_lfs_integrity_check: false) - Checks::ChangeAccess.new( + change_access = Checks::ChangeAccess.new( change, user_access: user_access, project: project, - skip_authorization: deploy_key?, skip_lfs_integrity_check: skip_lfs_integrity_check, - protocol: protocol - ).exec + protocol: protocol, + logger: logger + ) + + change_access.exec + rescue Checks::TimedLogger::TimeoutError + raise TimeoutError, logger.full_message end def deploy_key @@ -321,6 +361,10 @@ module Gitlab protected + def changes_list + @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes) + end + def user return @user if defined?(@user) diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb new file mode 100644 index 00000000000..a05a4baed82 --- /dev/null +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class CustomAction + attr_reader :payload, :message + + # Example of payload: + # + # { + # 'action' => 'geo_proxy_to_primary', + # 'data' => { + # 'api_endpoints' => %w{geo/proxy_git_push_ssh/info_refs geo/proxy_git_push_ssh/push}, + # 'gl_username' => user.username, + # 'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki) + # } + # } + # + def initialize(payload, message) + @payload = payload + @message = message + end + end + end +end diff --git a/lib/gitlab/git_access_result/success.rb b/lib/gitlab/git_access_result/success.rb new file mode 100644 index 00000000000..7bb9f24cb0e --- /dev/null +++ b/lib/gitlab/git_access_result/success.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module GitAccessResult + class Success + end + end +end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index a5b3902ebf4..0af91957fa8 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class GitAccessWiki < GitAccess ERROR_MESSAGES = { @@ -13,7 +15,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change, _options = {}) + def check_change_access! unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end diff --git a/lib/gitlab/git_logger.rb b/lib/gitlab/git_logger.rb index 9e02ccc0f44..dac4ddd320f 100644 --- a/lib/gitlab/git_logger.rb +++ b/lib/gitlab/git_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class GitLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 742118b76a8..426436c2164 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -1,23 +1,27 @@ +# frozen_string_literal: true + module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :project, :identifier, :changes + attr_reader :project, :identifier, :changes, :push_options - def initialize(project, identifier, changes) + def initialize(project, identifier, changes, push_options) @project = project @identifier = identifier @changes = deserialize_changes(changes) + @push_options = push_options end - def identify(revision) - super(identifier, project, revision) + def identify + super(identifier) end def changes_refs - return enum_for(:changes_refs) unless block_given? + return changes unless block_given? changes.each do |change| - oldrev, newrev, ref = change.strip.split(' ') + change.strip! + oldrev, newrev, ref = change.split(' ') yield oldrev, newrev, ref end @@ -26,13 +30,10 @@ module Gitlab private def deserialize_changes(changes) - changes = utf8_encode_changes(changes) - changes.lines + utf8_encode_changes(changes).each_line end def utf8_encode_changes(changes) - changes = changes.dup - changes.force_encoding('UTF-8') return changes if changes.valid_encoding? diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 2e3e4fc3f1f..3f13ebeb9d0 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitaly note: JV: does not need to be migrated, works without a repo. module Gitlab @@ -7,11 +9,15 @@ module Gitlab # # Returns true for a valid reference name, false otherwise def validate(ref_name) - return false if ref_name.start_with?('refs/heads/') - return false if ref_name.start_with?('refs/remotes/') + not_allowed_prefixes = %w(refs/heads/ refs/remotes/ -) + return false if ref_name.start_with?(*not_allowed_prefixes) + return false if ref_name == 'HEAD' - Gitlab::Utils.system_silent( - %W(#{Gitlab.config.git.bin_path} check-ref-format --branch #{ref_name})) + begin + Rugged::Reference.valid_name?("refs/heads/#{ref_name}") + rescue ArgumentError + return false + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 550294916a4..0ab53f8f706 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'base64' require 'gitaly' @@ -7,11 +9,6 @@ require 'grpc/health/v1/health_services_pb' module Gitlab module GitalyClient include Gitlab::Metrics::Methods - module MigrationStatus - DISABLED = 1 - OPT_IN = 2 - OPT_OUT = 3 - end class TooManyInvocationsError < StandardError attr_reader :call_site, :invocation_count, :max_call_stack @@ -23,21 +20,17 @@ module Gitlab stacks = most_invoked_stack.join('\n') if most_invoked_stack msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?" - msg << "\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks + msg = "#{msg}\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks super(msg) end end - SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 35 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze - # We have a mechanism to let GitLab automatically opt in to all Gitaly - # features. We want to be able to exclude some features from automatic - # opt-in. That is what EXPLICIT_OPT_IN_REQUIRED is for. - EXPLICIT_OPT_IN_REQUIRED = [Gitlab::GitalyClient::StorageSettings::DISK_ACCESS_DENIED_FLAG].freeze - MUTEX = Mutex.new class << self @@ -46,11 +39,6 @@ module Gitlab self.query_time = 0 - define_histogram :gitaly_migrate_call_duration_seconds do - docstring "Gitaly migration call execution timings" - base_labels gitaly_enabled: nil, feature: nil - end - define_histogram :gitaly_controller_action_duration_seconds do docstring "Gitaly endpoint histogram by controller and action combination" base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) @@ -63,11 +51,49 @@ module Gitlab @stubs[storage][name] ||= begin klass = stub_class(name) addr = stub_address(storage) - klass.new(addr, :this_channel_is_insecure) + creds = stub_creds(storage) + klass.new(addr, creds, interceptors: interceptors) end end end + def self.interceptors + return [] unless Gitlab::Tracing.enabled? + + [Gitlab::Tracing::GRPCInterceptor.instance] + end + private_class_method :interceptors + + def self.stub_cert_paths + cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"] + cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE + cert_paths + end + + def self.stub_certs + return @certs if @certs + + @certs = stub_cert_paths.flat_map do |cert_file| + File.read(cert_file).scan(PEM_REGEX).map do |cert| + begin + OpenSSL::X509::Certificate.new(cert).to_pem + rescue OpenSSL::OpenSSLError => e + Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) + nil + end + end.compact + end.uniq.join("\n") + end + + def self.stub_creds(storage) + if URI(address(storage)).scheme == 'tls' + GRPC::Core::ChannelCredentials.new stub_certs + else + :this_channel_is_insecure + end + end + def self.stub_class(name) if name == :health_check Grpc::Health::V1::Health::Stub @@ -77,9 +103,7 @@ module Gitlab end def self.stub_address(storage) - addr = address(storage) - addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp' - addr + address(storage).sub(%r{^tcp://|^tls://}, '') end def self.clear_stubs! @@ -101,15 +125,19 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix)) - raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix'" + unless URI(address).scheme.in?(%w(tcp unix tls)) + raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end address end def self.address_metadata(storage) - Base64.strict_encode64(JSON.dump({ storage => { 'address' => address(storage), 'token' => token(storage) } })) + Base64.strict_encode64(JSON.dump(storage => connection_data(storage))) + end + + def self.connection_data(storage) + { 'address' => address(storage), 'token' => token(storage) } end # All Gitaly RPC call sites should use GitalyClient.call. This method @@ -129,7 +157,6 @@ module Gitlab def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) start = Gitlab::Metrics::System.monotonic_time request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} - @current_call_id ||= SecureRandom.uuid enforce_gitaly_request_limits(:call) @@ -142,15 +169,13 @@ module Gitlab ensure duration = Gitlab::Metrics::System.monotonic_time - start - # Keep track, seperately, for the performance bar + # Keep track, separately, for the performance bar self.query_time += duration gitaly_controller_action_duration_seconds.observe( current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) - add_call_details(id: @current_call_id, feature: service, duration: duration, request: request_hash) - - @current_call_id = nil + add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) end def self.handle_grpc_unavailable!(ex) @@ -179,10 +204,29 @@ module Gitlab end private_class_method :current_transaction_labels + # For some time related tasks we can't rely on `Time.now` since it will be + # affected by Timecop in some tests, and the clock of some gitaly-related + # components (grpc's c-core and gitaly server) use system time instead of + # timecop's time, so tests will fail. + # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent + # timecop. + def self.real_time + Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + end + private_class_method :real_time + + def self.authorization_token(storage) + token = token(storage).to_s + issued_at = real_time.to_i.to_s + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + + "v2.#{hmac}.#{issued_at}" + end + private_class_method :authorization_token + def self.request_kwargs(storage, timeout, remote_storage: nil) - encoded_token = Base64.strict_encode64(token(storage).to_s) metadata = { - 'authorization' => "Bearer #{encoded_token}", + 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME } @@ -190,6 +234,9 @@ module Gitlab feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage + metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id + + metadata.merge!(server_feature_flags) result = { metadata: metadata } @@ -198,17 +245,20 @@ module Gitlab return result unless timeout > 0 - # Do not use `Time.now` for deadline calculation, since it - # will be affected by Timecop in some tests, but grpc's c-core - # uses system time instead of timecop's time, so tests will fail - # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will - # circumvent timecop - deadline = Time.at(Process.clock_gettime(Process::CLOCK_REALTIME)) + timeout + deadline = real_time + timeout result[:deadline] = deadline result end + SERVER_FEATURE_FLAGS = %w[].freeze + + def self.server_feature_flags + SERVER_FEATURE_FLAGS.map do |f| + ["gitaly-feature-#{f.tr('_', '-')}", feature_enabled?(f).to_s] + end.to_h + end + def self.token(storage) params = Gitlab.config.repositories.storages[storage] raise "storage not found: #{storage.inspect}" if params.nil? @@ -216,77 +266,14 @@ module Gitlab params['gitaly_token'].presence || Gitlab.config.gitaly['token'] end - # Evaluates whether a feature toggle is on or off - def self.feature_enabled?(feature_name, status: MigrationStatus::OPT_IN) - # Disabled features are always off! - return false if status == MigrationStatus::DISABLED - - feature = Feature.get("gitaly_#{feature_name}") - - # If the feature has been set, always evaluate - if Feature.persisted?(feature) - if feature.percentage_of_time_value > 0 - # Probabilistically enable this feature - return Random.rand() * 100 < feature.percentage_of_time_value - end - - return feature.enabled? - end - - # If the feature has not been set, the default depends - # on it's status - case status - when MigrationStatus::OPT_OUT - true - when MigrationStatus::OPT_IN - opt_into_all_features? && !EXPLICIT_OPT_IN_REQUIRED.include?(feature_name) - else - false - end - end - - # opt_into_all_features? returns true when the current environment - # is one in which we opt into features automatically - def self.opt_into_all_features? - Rails.env.development? || ENV["GITALY_FEATURE_DEFAULT_ON"] == "1" - end - private_class_method :opt_into_all_features? - - def self.migrate(feature, status: MigrationStatus::OPT_IN) - # Enforce limits at both the `migrate` and `call` sites to ensure that - # problems are not hidden by a feature being disabled - enforce_gitaly_request_limits(:migrate) - - is_enabled = feature_enabled?(feature, status: status) - metric_name = feature.to_s - metric_name += "_gitaly" if is_enabled - - Gitlab::Metrics.measure(metric_name) do - # Some migrate calls wrap other migrate calls - allow_n_plus_1_calls do - feature_stack = Thread.current[:gitaly_feature_stack] ||= [] - feature_stack.unshift(feature) - begin - start = Gitlab::Metrics::System.monotonic_time - @current_call_id = SecureRandom.uuid - call_details = { id: @current_call_id } - yield is_enabled - ensure - total_time = Gitlab::Metrics::System.monotonic_time - start - gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) - feature_stack.shift - Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? - - add_call_details(call_details.merge(feature: feature, duration: total_time)) - end - end - end + def self.feature_enabled?(feature_name) + Feature.enabled?("gitaly_#{feature_name}") end # Ensures that Gitaly is not being abuse through n+1 misuse etc def self.enforce_gitaly_request_limits(call_site) # Only count limits in request-response environments (not sidekiq for example) - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") @@ -310,7 +297,7 @@ module Gitlab end def self.allow_n_plus_1_calls - return yield unless RequestStore.active? + return yield unless Gitlab::SafeRequestStore.active? begin increment_call_count(:gitaly_call_count_exception_block_depth) @@ -321,63 +308,44 @@ module Gitlab end def self.get_call_count(key) - RequestStore.store[key] || 0 + Gitlab::SafeRequestStore[key] || 0 end private_class_method :get_call_count def self.increment_call_count(key) - RequestStore.store[key] ||= 0 - RequestStore.store[key] += 1 + Gitlab::SafeRequestStore[key] ||= 0 + Gitlab::SafeRequestStore[key] += 1 end private_class_method :increment_call_count def self.decrement_call_count(key) - RequestStore.store[key] -= 1 + Gitlab::SafeRequestStore[key] -= 1 end private_class_method :decrement_call_count - # Returns an estimate of the number of Gitaly calls made for this - # request + # Returns the of the number of Gitaly calls made for this request def self.get_request_count - return 0 unless RequestStore.active? - - gitaly_migrate_count = get_call_count("gitaly_migrate_actual") - gitaly_call_count = get_call_count("gitaly_call_actual") - - # Using the maximum of migrate and call_count will provide an - # indicator of how many Gitaly calls will be made, even - # before a feature is enabled. This provides us with a single - # metric, but not an exact number, but this tradeoff is acceptable - if gitaly_migrate_count > gitaly_call_count - gitaly_migrate_count - else - gitaly_call_count - end + get_call_count("gitaly_call_actual") end def self.reset_counts - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? - %w[migrate call].each do |call_site| - RequestStore.store["gitaly_#{call_site}_actual"] = 0 - RequestStore.store["gitaly_#{call_site}_permitted"] = 0 - end + Gitlab::SafeRequestStore["gitaly_call_actual"] = 0 + Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 end def self.add_call_details(details) - id = details.delete(:id) + return unless Gitlab::SafeRequestStore[:peek_enabled] - return unless id && RequestStore.active? && RequestStore.store[:peek_enabled] - - RequestStore.store['gitaly_call_details'] ||= {} - RequestStore.store['gitaly_call_details'][id] ||= {} - RequestStore.store['gitaly_call_details'][id].merge!(details) + Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] + Gitlab::SafeRequestStore['gitaly_call_details'] << details end def self.list_call_details - return {} unless RequestStore.active? && RequestStore.store[:peek_enabled] + return [] unless Gitlab::SafeRequestStore[:peek_enabled] - RequestStore.store['gitaly_call_details'] || {} + Gitlab::SafeRequestStore['gitaly_call_details'] || [] end def self.expected_server_version @@ -385,13 +353,13 @@ module Gitlab path.read.chomp end - def self.timestamp(t) - Google::Protobuf::Timestamp.new(seconds: t.to_i) + def self.timestamp(time) + Google::Protobuf::Timestamp.new(seconds: time.to_i) end # The default timeout on all Gitaly calls def self.default_timeout - return 0 if Sidekiq.server? + return no_timeout if Sidekiq.server? timeout(:gitaly_timeout_default) end @@ -404,6 +372,10 @@ module Gitlab timeout(:gitaly_timeout_medium) end + def self.no_timeout + 0 + end + def self.timeout(timeout_name) Gitlab::CurrentSettings.current_application_settings[timeout_name] end @@ -411,22 +383,22 @@ module Gitlab # Count a stack. Used for n+1 detection def self.count_stack - return unless RequestStore.active? + return unless Gitlab::SafeRequestStore.active? - stack_string = caller.drop(1).join("\n") + stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") - RequestStore.store[:stack_counter] ||= Hash.new + Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new - count = RequestStore.store[:stack_counter][stack_string] || 0 - RequestStore.store[:stack_counter][stack_string] = count + 1 + count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0 + Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1 end private_class_method :count_stack # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection def self.max_call_count - return 0 unless RequestStore.active? + return 0 unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return 0 unless stack_counter stack_counter.values.max @@ -435,9 +407,9 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless RequestStore.active? + return nil unless Gitlab::SafeRequestStore.active? - stack_counter = RequestStore.store[:stack_counter] + stack_counter = Gitlab::SafeRequestStore[:stack_counter] return nil unless stack_counter max = max_call_count diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb index 198a1de91c7..3f1a0ef4888 100644 --- a/lib/gitlab/gitaly_client/attributes_bag.rb +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient # This module expects an `ATTRS` const to be defined on the subclass diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 28554208984..39547328210 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class BlobService @@ -13,9 +15,9 @@ module Gitlab oid: oid, limit: limit ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) - data = '' + data = [] blob = nil response.each do |msg| if blob.nil? @@ -27,6 +29,8 @@ module Gitlab return nil if blob.oid.blank? + data = data.join + Gitlab::Git::Blob.new( id: blob.oid, size: blob.size, @@ -43,7 +47,7 @@ module Gitlab blob_ids: blob_ids ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end @@ -66,13 +70,13 @@ module Gitlab :blob_service, :get_blobs, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) GitalyClient::BlobsStitcher.new(response) end - def get_new_lfs_pointers(revision, limit, not_in) + def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) request = Gitaly::GetNewLFSPointersRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), @@ -85,7 +89,20 @@ module Gitlab request.not_in_refs += not_in end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_new_lfs_pointers, + request, + timeout: timeout + ) map_lfs_pointers(response) end @@ -96,7 +113,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index 5ca592ff812..01bab854082 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class BlobsStitcher diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb new file mode 100644 index 00000000000..3e8d6a773ca --- /dev/null +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class CleanupService + attr_reader :repository, :gitaly_repo, :storage + + # 'repository' is a Gitlab::Git::Repository + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def apply_bfg_object_map(io) + first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo) + + enum = Enumerator.new do |y| + y.yield first_request + + while data = io.read(RepositoryService::MAX_MSG_SIZE) + y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + break if io&.eof? + end + end + + GitalyClient.call( + storage, + :cleanup_service, + :apply_bfg_object_map, + enum, + timeout: GitalyClient.no_timeout + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index a4cc64de80d..4e46cb9f05c 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class CommitService @@ -70,12 +72,19 @@ module Gitlab def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) response.flat_map { |msg| msg.deltas } end def tree_entry(ref, path, limit = nil) + if Pathname.new(path).cleanpath.to_s.start_with?('../') + # The TreeEntry RPC should return an empty reponse in this case but in + # Gitaly 0.107.0 and earlier we get an exception instead. This early return + # saves us a Gitaly roundtrip while also avoiding the exception. + return + end + request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), @@ -86,7 +95,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil - data = '' + data = [] response.each do |msg| if entry.nil? entry = msg @@ -96,7 +105,7 @@ module Gitlab data << msg.data end - entry.data = data + entry.data = data.join entry unless entry.oid.blank? end @@ -141,6 +150,24 @@ module Gitlab GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end + def list_last_commits_for_tree(revision, path, offset: 0, limit: 25) + request = Gitaly::ListLastCommitsForTreeRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision), + path: encode_binary(path.to_s), + offset: offset, + limit: limit + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + + response.each_with_object({}) do |gitaly_response, hsh| + gitaly_response.commits.each do |commit_for_tree| + hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + end + end + end + def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, @@ -165,6 +192,17 @@ module Gitlab consume_commits_response(response) end + def diff_stats(left_commit_sha, right_commit_sha) + request = Gitaly::DiffStatsRequest.new( + repository: @gitaly_repo, + left_commit_id: left_commit_sha, + right_commit_id: right_commit_sha + ) + + response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response.flat_map(&:stats) + end + def find_all_commits(opts = {}) request = Gitaly::FindAllCommitsRequest.new( repository: @gitaly_repo, @@ -179,6 +217,8 @@ module Gitlab end def list_commits_by_oid(oids) + return [] if oids.empty? + request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) @@ -216,31 +256,33 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) - response.reduce("") { |memo, msg| memo << msg.data } + response.reduce([]) { |memo, msg| memo << msg.data }.join end def find_commit(revision) - if RequestStore.active? - # We don't use RequeStstore.fetch(key) { ... } directly because `revision` - # can be a branch name, so we can't use it as a key as it could point - # to another commit later on (happens a lot in tests). + if Gitlab::SafeRequestStore.active? + # We don't use Gitlab::SafeRequestStore.fetch(key) { ... } directly + # because `revision` can be a branch name, so we can't use it as a key + # as it could point to another commit later on (happens a lot in + # tests). key = { storage: @gitaly_repo.storage_name, relative_path: @gitaly_repo.relative_path, commit_id: revision } - return RequestStore[key] if RequestStore.exist?(key) + return Gitlab::SafeRequestStore[key] if Gitlab::SafeRequestStore.exist?(key) commit = call_find_commit(revision) return unless commit key[:commit_id] = commit.id - RequestStore[key] = commit + Gitlab::SafeRequestStore[key] = commit else call_find_commit(revision) end end + # rubocop: disable CodeReuse/ActiveRecord def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, @@ -250,6 +292,7 @@ module Gitlab response.sum(&:data) end + # rubocop: enable CodeReuse/ActiveRecord def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( @@ -293,7 +336,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum) + response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) response.flat_map do |msg| msg.shas.map { |sha| EncodingHelper.encode!(sha) } @@ -304,8 +347,8 @@ module Gitlab request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request) - signature = ''.b - signed_text = ''.b + signature = +''.b + signed_text = +''.b response.each do |message| signature << message.signature @@ -315,13 +358,15 @@ module Gitlab return if signature.blank? && signed_text.blank? [signature, signed_text] + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) - signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] } + signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_commit_id = nil response.each do |message| @@ -332,13 +377,15 @@ module Gitlab end signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_messages(commit_ids) request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) - messages = Hash.new { |h, k| h[k] = ''.b } + messages = Hash.new { |h, k| h[k] = +''.b } current_commit_id = nil response.each do |rpc_message| @@ -355,8 +402,8 @@ module Gitlab def call_commit_diff(request_params, options = {}) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) - request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params[:collapse_diffs] = !options.fetch(:expanded, true) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) @@ -386,8 +433,8 @@ module Gitlab end end - def encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + def encode_repeated(array) + Google::Protobuf::RepeatedField.new(:bytes, array.map { |s| encode_binary(s) } ) end def call_find_commit(revision) diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index c275a065bce..0e00f6e8c44 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class ConflictFilesStitcher @@ -17,7 +19,7 @@ module Gitlab current_file = file_from_gitaly_header(gitaly_file.header) else - current_file.raw_content << gitaly_file.content + current_file.raw_content = "#{current_file.raw_content}#{gitaly_file.content}" end end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index e14734495a8..6304f998563 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class ConflictsService @@ -25,10 +27,12 @@ module Gitlab def conflicts? list_conflict_files.any? - rescue GRPC::FailedPrecondition - # The server raises this exception when it encounters ConflictSideMissing, which - # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent - # file in one of the trees. + rescue GRPC::FailedPrecondition, GRPC::Unknown + # The server raises FailedPrecondition when it encounters + # ConflictSideMissing, which means a conflict exists but its `theirs` or + # `ours` data is nil due to a non-existent file in one of the trees. + # + # GRPC::Unknown comes from Rugged::ReferenceError and Rugged::OdbError. true end @@ -46,7 +50,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.medium_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index d98a0ce988f..dd192ccde1a 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -1,7 +1,9 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class Diff - ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed).freeze + ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed too_large).freeze include AttributesBag end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index da243ee2d1a..98d327a7329 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class DiffStitcher @@ -20,7 +22,7 @@ module Gitlab current_diff = GitalyClient::Diff.new(diff_params) else - current_diff.patch += diff_msg.raw_patch_data + current_diff.patch = "#{current_diff.patch}#{diff_msg.raw_patch_data}" end if diff_msg.end_of_patch diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb index 6c1213f5e20..0c495f60633 100644 --- a/lib/gitlab/gitaly_client/health_check_service.rb +++ b/lib/gitlab/gitaly_client/health_check_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class HealthCheckService diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index bd7c345ac01..f0be3cbebd2 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class NamespaceService @@ -8,31 +10,31 @@ module Gitlab def exists?(name) request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:namespace_exists, request).exists + gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout).exists end def add(name) request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:add_namespace, request) + gitaly_client_call(:add_namespace, request, timeout: GitalyClient.fast_timeout) end def remove(name) request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:remove_namespace, request) + gitaly_client_call(:remove_namespace, request, timeout: nil) end def rename(from, to) request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) - gitaly_client_call(:rename_namespace, request) + gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout) end private - def gitaly_client_call(type, request) - GitalyClient.call(@storage, :namespace_service, type, request) + def gitaly_client_call(type, request, timeout: nil) + GitalyClient.call(@storage, :namespace_service, type, request, timeout: timeout) end end end diff --git a/lib/gitlab/gitaly_client/notification_service.rb b/lib/gitlab/gitaly_client/notification_service.rb index 326e6f7dafc..873c3e4086d 100644 --- a/lib/gitlab/gitaly_client/notification_service.rb +++ b/lib/gitlab/gitaly_client/notification_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class NotificationService diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb new file mode 100644 index 00000000000..6e7ede5fd18 --- /dev/null +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class ObjectPoolService + attr_reader :object_pool, :storage + + def initialize(object_pool) + @object_pool = object_pool.gitaly_object_pool + @storage = object_pool.storage + end + + def create(repository) + request = Gitaly::CreateObjectPoolRequest.new( + object_pool: object_pool, + origin: repository.gitaly_repository) + + GitalyClient.call(storage, :object_pool_service, :create_object_pool, request) + end + + def delete + request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool) + + GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request) + end + + def link_repository(repository) + request = Gitaly::LinkRepositoryToObjectPoolRequest.new( + object_pool: object_pool, + repository: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, + request, timeout: GitalyClient.fast_timeout) + end + + def unlink_repository(repository) + request = Gitaly::UnlinkRepositoryFromObjectPoolRequest.new( + object_pool: object_pool, + repository: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :unlink_repository_from_object_pool, + request, timeout: GitalyClient.fast_timeout) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 44b0e517bf0..22d2d149e65 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class OperationService @@ -17,10 +19,10 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end end @@ -33,9 +35,9 @@ module Gitlab message: encode_binary(message.to_s) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists raise Gitlab::Git::Repository::TagExistsError end @@ -56,7 +58,7 @@ module Gitlab :user_create_branch, request) if response.pre_receive_error.present? - raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) + raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) end branch = response.branch @@ -64,6 +66,24 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + rescue GRPC::FailedPrecondition => ex + raise Gitlab::Git::Repository::InvalidRef, ex + end + + def user_update_branch(branch_name, user, newrev, oldrev) + request = Gitaly::UserUpdateBranchRequest.new( + repository: @gitaly_repo, + branch_name: encode_binary(branch_name), + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + newrev: encode_binary(newrev), + oldrev: encode_binary(oldrev) + ) + + response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request) + + if pre_receive_error = response.pre_receive_error.presence + raise Gitlab::Git::PreReceiveError, pre_receive_error + end end def user_delete_branch(branch_name, user) @@ -76,7 +96,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end end @@ -106,7 +126,7 @@ module Gitlab second_response = response_enum.next if second_response.pre_receive_error.present? - raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error + raise Gitlab::Git::PreReceiveError, second_response.pre_receive_error end branch_update = second_response.branch_update @@ -126,13 +146,16 @@ module Gitlab branch: encode_binary(target_branch) ) - branch_update = GitalyClient.call( + response = GitalyClient.call( @repository.storage, :operation_service, :user_ff_branch, request - ).branch_update - Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + ) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::CommitError, e end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) @@ -175,7 +198,7 @@ module Gitlab ) if response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + raise Gitlab::Git::PreReceiveError, response.pre_receive_error elsif response.git_error.presence raise Gitlab::Git::Repository::GitError, response.git_error else @@ -209,6 +232,32 @@ module Gitlab response.squash_sha end + def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) + request = Gitaly::UserUpdateSubmoduleRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_sha: commit_sha, + branch: encode_binary(branch), + submodule: encode_binary(submodule), + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_update_submodule, + request + ) + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.present? + raise Gitlab::Git::CommitError, response.commit_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) @@ -242,7 +291,7 @@ module Gitlab :user_commit_files, req_enum, remote_storage: start_repository.storage) if (pre_receive_error = response.pre_receive_error.presence) - raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error + raise Gitlab::Git::PreReceiveError, pre_receive_error end if (index_error = response.index_error.presence) @@ -252,6 +301,29 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + def user_commit_patches(user, branch_name, patches) + header = Gitaly::UserApplyPatchRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + target_branch: encode_binary(branch_name) + ) + reader = binary_stringio(patches) + + chunks = Enumerator.new do |chunk| + chunk.yield Gitaly::UserApplyPatchRequest.new(header: header) + + until reader.eof? + patch_chunk = reader.read(MAX_MSG_SIZE) + + chunk.yield(Gitaly::UserApplyPatchRequest.new(patches: patch_chunk)) + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) @@ -272,7 +344,8 @@ module Gitlab :operation_service, :"user_#{rpc}", request, - remote_storage: start_repository.storage + remote_storage: start_repository.storage, + timeout: GitalyClient.medium_timeout ) handle_cherry_pick_or_revert_response(response) @@ -280,14 +353,14 @@ module Gitlab def handle_cherry_pick_or_revert_response(response) if response.pre_receive_error.presence - raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + raise Gitlab::Git::PreReceiveError, response.pre_receive_error elsif response.commit_error.presence raise Gitlab::Git::CommitError, response.commit_error elsif response.create_tree_error.presence raise Gitlab::Git::Repository::CreateTreeError, response.create_tree_error - else - Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end def user_commit_files_request_header( @@ -311,7 +384,9 @@ module Gitlab action: action[:action].upcase.to_sym, file_path: encode_binary(action[:file_path]), previous_path: encode_binary(action[:previous_path]), - base64_content: action[:encoding] == 'base64' + base64_content: action[:encoding] == 'base64', + execute_filemode: !!action[:execute_filemode], + infer_content: !!action[:infer_content] ) rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" diff --git a/lib/gitlab/gitaly_client/queue_enumerator.rb b/lib/gitlab/gitaly_client/queue_enumerator.rb index b8018029552..3a412102abe 100644 --- a/lib/gitlab/gitaly_client/queue_enumerator.rb +++ b/lib/gitlab/gitaly_client/queue_enumerator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class QueueEnumerator diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 3ac46be6208..d5633d167ac 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class RefService @@ -12,37 +14,44 @@ module Gitlab def branches request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end + def remote_branches(remote_name) + request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name) + response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request) + + consume_find_all_remote_branches_response(remote_name, response) + end + def merged_branches(branch_names = []) request = Gitaly::FindAllBranchesRequest.new( repository: @gitaly_repo, merged_only: true, merged_branches: branch_names.map { |s| encode_binary(s) } ) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -56,6 +65,42 @@ module Gitlab encode!(response.name.dup) end + def list_new_commits(newrev) + request = Gitaly::ListNewCommitsRequest.new( + repository: @gitaly_repo, + commit_id: newrev + ) + + response = GitalyClient + .call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout) + + commits = [] + response.each do |msg| + msg.commits.each do |c| + commits << Gitlab::Git::Commit.new(@repository, c) + end + end + + commits + end + + def list_new_blobs(newrev, limit = 0) + request = Gitaly::ListNewBlobsRequest.new( + repository: @gitaly_repo, + commit_id: newrev, + limit: limit + ) + + response = GitalyClient + .call(@storage, :ref_service, :list_new_blobs, request, timeout: GitalyClient.medium_timeout) + + response.flat_map do |msg| + # Returns an Array of Gitaly::NewBlobObject objects + # Available methods are: #size, #oid and #path + msg.new_blob_objects + end + end + def count_tag_names tag_names.count end @@ -67,19 +112,19 @@ module Gitlab def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) consume_find_local_branches_response(response) end def tags request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) consume_tags_response(response) end def ref_exists?(ref_name) request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) - response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) + response = GitalyClient.call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) response.value rescue GRPC::InvalidArgument => e raise ArgumentError, e.message @@ -91,7 +136,7 @@ module Gitlab name: encode_binary(branch_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) + response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) branch = response.branch return unless branch @@ -140,7 +185,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end @@ -153,7 +198,7 @@ module Gitlab limit: limit ) - stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(stream, :tag_names) end @@ -166,16 +211,16 @@ module Gitlab limit: limit ) - stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(stream, :branch_names) end def get_tag_messages(tag_ids) request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids) - response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) - messages = Hash.new { |h, k| h[k] = ''.b } + messages = Hash.new { |h, k| h[k] = +''.b } current_tag_id = nil response.each do |rpc_message| @@ -224,6 +269,18 @@ module Gitlab end end + def consume_find_all_remote_branches_response(remote_name, response) + remote_name += '/' unless remote_name.ends_with?('/') + + response.flat_map do |message| + message.branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + branch_name = branch.name.sub(remote_name, '') + Gitlab::Git::Branch.new(@repository, branch_name, branch.target_commit.id, target_commit) + end + end + end + def consume_tags_response(response) response.flat_map do |message| message.tags.map { |gitaly_tag| Gitlab::Git::Tag.new(@repository, gitaly_tag) } diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index f2d699d9dfb..81fac37ee68 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -1,6 +1,10 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class RemoteService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def self.exists?(remote_url) @@ -28,13 +32,13 @@ module Gitlab mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) ) - GitalyClient.call(@storage, :remote_service, :add_remote, request) + GitalyClient.call(@storage, :remote_service, :add_remote, request, timeout: GitalyClient.fast_timeout) end def remove_remote(name) request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) - response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.fast_timeout) response.result end @@ -52,13 +56,30 @@ module Gitlab response.result end - def update_remote_mirror(ref_name, only_branches_matching) + def find_remote_root_ref(remote_name) + request = Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + + response = GitalyClient.call(@storage, :remote_service, + :find_remote_root_ref, request) + + encode_utf8(response.ref) + end + + def update_remote_mirror(ref_name, only_branches_matching, ssh_key: nil, known_hosts: nil) req_enum = Enumerator.new do |y| - y.yield Gitaly::UpdateRemoteMirrorRequest.new( + first_request = Gitaly::UpdateRemoteMirrorRequest.new( repository: @gitaly_repo, ref_name: ref_name ) + first_request.ssh_key = ssh_key if ssh_key.present? + first_request.known_hosts = known_hosts if known_hosts.present? + + y.yield(first_request) + current_size = 0 slices = only_branches_matching.slice_before do |branch_name| diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index ee01f5a5bd9..8a1abfbf874 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class RepositoryService @@ -21,7 +23,7 @@ module Gitlab def cleanup request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :cleanup, request) + GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout) end def garbage_collect(create_bitmap) @@ -41,22 +43,24 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request) + response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.medium_timeout) response.size end def apply_gitattributes(revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) - GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) + GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) + rescue GRPC::InvalidArgument => ex + raise Gitlab::Git::Repository::InvalidRef, ex end def info_attributes request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request) - response.each_with_object("") do |message, attributes| + response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) + response.each_with_object([]) do |message, attributes| attributes << message.attributes - end + end.join end def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) @@ -65,7 +69,7 @@ module Gitlab no_tags: no_tags, timeout: timeout, no_prune: !prune ) - if ssh_auth&.ssh_import? + if ssh_auth&.ssh_mirror_url? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? request.ssh_key = ssh_auth.ssh_private_key end @@ -80,7 +84,7 @@ module Gitlab def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :create_repository, request) + GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout) end def has_local_branches? @@ -96,7 +100,7 @@ module Gitlab revisions: revisions.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) + response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) response.base.presence end @@ -196,42 +200,38 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.no_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) - request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo) - enum = Enumerator.new do |y| - File.open(bundle_path, 'rb') do |f| - while data = f.read(MAX_MSG_SIZE) - request.data = data - - y.yield request - - request = Gitaly::CreateRepositoryFromBundleRequest.new - end - end - end - - GitalyClient.call( - @storage, - :repository_service, + gitaly_repo_stream_request( + bundle_path, :create_repository_from_bundle, - enum, - timeout: GitalyClient.default_timeout + Gitaly::CreateRepositoryFromBundleRequest, + GitalyClient.no_timeout + ) + end + + def restore_custom_hooks(custom_hooks_path) + gitaly_repo_stream_request( + custom_hooks_path, + :restore_custom_hooks, + Gitaly::RestoreCustomHooksRequest, + GitalyClient.default_timeout ) end @@ -247,37 +247,54 @@ module Gitlab :repository_service, :create_repository_from_snapshot, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.no_timeout ) end - def write_ref(ref_path, ref, old_ref, shell) + def write_ref(ref_path, ref, old_ref) request = Gitaly::WriteRefRequest.new( repository: @gitaly_repo, ref: ref_path.b, - revision: ref.b, - shell: shell + revision: ref.b ) request.old_revision = old_ref.b unless old_ref.nil? - response = GitalyClient.call(@storage, :repository_service, :write_ref, request) + GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) + end - raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present? + def set_config(entries) + return if entries.empty? - true + request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo) + entries.each do |key, value| + request.entries << build_set_config_entry(key, value) + end + + GitalyClient.call( + @storage, + :repository_service, + :set_config, + request, + timeout: GitalyClient.fast_timeout + ) + + nil end - def write_config(full_path:) - request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path) - response = GitalyClient.call( + def delete_config(keys) + return if keys.empty? + + request = Gitaly::DeleteConfigRequest.new(repository: @gitaly_repo, keys: keys) + + GitalyClient.call( @storage, :repository_service, - :write_config, + :delete_config, request, timeout: GitalyClient.fast_timeout ) - raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? + nil end def license_short_name @@ -290,7 +307,7 @@ module Gitlab def calculate_checksum request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e raise Gitlab::Git::Repository::InvalidRepository.new(e) @@ -299,18 +316,78 @@ module Gitlab def raw_changes_between(from, to) request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to) - GitalyClient.call(@storage, :repository_service, :get_raw_changes, request) + GitalyClient.call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) end def search_files_by_name(ref, query) request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) GitalyClient.call(@storage, :repository_service, :search_files_by_content, request).flat_map(&:matches) end + + private + + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we received an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + enum = Enumerator.new do |y| + File.open(file_path, 'rb') do |f| + while data = f.read(MAX_MSG_SIZE) + request.data = data + + y.yield request + request = request_class.new + end + end + end + + GitalyClient.call( + @storage, + :repository_service, + rpc_name, + enum, + timeout: timeout + ) + end + + def build_set_config_entry(key, value) + entry = Gitaly::SetConfigRequest::Entry.new(key: key) + + case value + when String + entry.value_str = value + when Integer + entry.value_int32 = value + when TrueClass, FalseClass + entry.value_bool = value + else + raise InvalidArgument, "invalid git config value: #{value.inspect}" + end + + entry + end end end end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 2e1076d1f66..0ade6942db9 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient # Meant for extraction of server data, and later maybe to perform misc task @@ -9,7 +11,7 @@ module Gitlab end def info - GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new) + GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout) end end end diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb index eb0e910665b..4edcb0b8ba9 100644 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class StorageService @@ -5,6 +7,14 @@ module Gitlab @storage = storage end + # Returns all directories in the git storage directory, lexically ordered + def list_directories(depth: 1) + request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) + + GitalyClient.call(@storage, :storage_service, :list_directories, request) + .flat_map(&:paths) + end + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. def delete_all_repositories request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 02fcb413abd..754cccb6b3f 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient # This is a chokepoint that is meant to help us stop remove all places @@ -13,7 +15,7 @@ module Gitlab Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. - If you're using the Gitlab Development Kit, you can update your configuration running `gdk reconfigure`. + If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. MSG # This class will give easily recognizable NoMethodErrors @@ -60,8 +62,8 @@ module Gitlab private - def method_missing(m, *args, &block) - @hash.public_send(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(msg, *args, &block) + @hash.public_send(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 9c19c51d412..dce5d6a8ad0 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient module Util diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb index 47c60c92484..ef2b23732d1 100644 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ b/lib/gitlab/gitaly_client/wiki_file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class WikiFile diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index a02d15db5dd..757a429fb8a 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitalyClient class WikiPage diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 2dfe055a496..2b3d622af4d 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'stringio' module Gitlab @@ -69,7 +71,7 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request) + GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request, timeout: GitalyClient.medium_timeout) end def find_page(title:, version: nil, dir: nil) @@ -80,14 +82,14 @@ module Gitlab directory: encode_binary(dir) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) wiki_page_from_iterator(response) end - def get_all_pages - request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request) + def get_all_pages(limit: 0) + request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo, limit: limit) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) pages = [] loop do @@ -110,10 +112,10 @@ module Gitlab repository: @gitaly_repo, page_path: encode_binary(page_path), page: options[:page] || 1, - per_page: options[:per_page] || Gollum::Page.per_page + per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION ) - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) versions = [] stream.each do |message| @@ -132,14 +134,14 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout) wiki_file = nil response.each do |message| next unless message.name.present? || wiki_file if wiki_file - wiki_file.raw_data << message.raw_data + wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}" else wiki_file = GitalyClient::WikiFile.new(message.to_h) # All gRPC strings in a response are frozen, so we get @@ -160,7 +162,7 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request) - response.reduce("") { |memo, msg| memo << msg.data } + response.reduce([]) { |memo, msg| memo << msg.data }.join end private diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 65b5e30c70f..14a6d6443ec 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GithubImport def self.refmap @@ -10,24 +12,6 @@ module Gitlab Client.new(token_to_use, parallel: parallel) end - # Inserts a raw row and returns the ID of the inserted row. - # - # attributes - The attributes/columns to set. - # relation - An ActiveRecord::Relation to use for finding the ID of the row - # when using MySQL. - def self.insert_and_return_id(attributes, relation) - # We use bulk_insert here so we can bypass any queries executed by - # callbacks or validation rules, as doing this wouldn't scale when - # importing very large projects. - result = Gitlab::Database - .bulk_insert(relation.table_name, [attributes], return_ids: true) - - # MySQL doesn't support returning the IDs of a bulk insert in a way that - # is not a pain, so in this case we'll issue an extra query instead. - result.first || - relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first - end - # Returns the ID of the ghost user. def self.ghost_user_id key = 'github-import/ghost-user-id' diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 8274f37d358..d562958e955 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -13,7 +13,7 @@ module Gitlab @note = note @project = project @client = client - @user_finder = UserFinder.new(project, client) + @user_finder = GithubImport::UserFinder.new(project, client) end def execute diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 31fefebf787..656d46b6a7d 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class IssueImporter + include Gitlab::Import::DatabaseHelpers + attr_reader :project, :issue, :client, :user_finder, :milestone_finder, :issuable_finder @@ -19,7 +21,7 @@ module Gitlab @issue = issue @project = project @client = client - @user_finder = UserFinder.new(project, client) + @user_finder = GithubImport::UserFinder.new(project, client) @milestone_finder = MilestoneFinder.new(project) @issuable_finder = GithubImport::IssuableFinder.new(project, issue) end @@ -55,7 +57,7 @@ module Gitlab updated_at: issue.updated_at } - GithubImport.insert_and_return_id(attributes, project.issues) + insert_and_return_id(attributes, project.issues) rescue ActiveRecord::InvalidForeignKey # It's possible the project has been deleted since scheduling this # job. In this case we'll just skip creating the issue. diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index a73033d35ba..80246fa1b77 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_labels = project.labels.pluck(:title).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Label, build_labels) diff --git a/lib/gitlab/github_import/importer/lfs_object_importer.rb b/lib/gitlab/github_import/importer/lfs_object_importer.rb new file mode 100644 index 00000000000..195383fd3e9 --- /dev/null +++ b/lib/gitlab/github_import/importer/lfs_object_importer.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LfsObjectImporter + attr_reader :lfs_object, :project + + # lfs_object - An instance of `Gitlab::GithubImport::Representation::LfsObject`. + # project - An instance of `Project`. + def initialize(lfs_object, project, _) + @lfs_object = lfs_object + @project = project + end + + def lfs_download_object + LfsDownloadObject.new(oid: lfs_object.oid, size: lfs_object.size, link: lfs_object.link) + end + + def execute + Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb new file mode 100644 index 00000000000..6046e30d4ef --- /dev/null +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class LfsObjectsImporter + include ParallelScheduling + + def importer_class + LfsObjectImporter + end + + def representation_class + Representation::LfsObject + end + + def sidekiq_worker_class + ImportLfsObjectWorker + end + + def collection_method + :lfs_objects + end + + def each_object_to_import + lfs_objects = Projects::LfsPointers::LfsImportService.new(project).execute + + lfs_objects.each do |object| + yield object + end + rescue StandardError => e + Rails.logger.error("The Lfs import process failed. #{e.message}") + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index c53480e828a..87cf2c8b598 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_milestones = project.milestones.pluck(:iid).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Milestone, build_milestones) diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index c890f2df360..2b06d1b3baf 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -13,7 +13,7 @@ module Gitlab @note = note @project = project @client = client - @user_finder = UserFinder.new(project, client) + @user_finder = GithubImport::UserFinder.new(project, client) end def execute diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 49d859f9624..ae7c4cf1b38 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class PullRequestImporter + include Gitlab::Import::MergeRequestHelpers + attr_reader :pull_request, :project, :client, :user_finder, :milestone_finder, :issuable_finder @@ -15,75 +17,56 @@ module Gitlab @pull_request = pull_request @project = project @client = client - @user_finder = UserFinder.new(project, client) + @user_finder = GithubImport::UserFinder.new(project, client) @milestone_finder = MilestoneFinder.new(project) @issuable_finder = GithubImport::IssuableFinder.new(project, pull_request) end def execute - if (mr_id = create_merge_request) - issuable_finder.cache_database_id(mr_id) + mr, already_exists = create_merge_request + + if mr + insert_git_data(mr, already_exists) + issuable_finder.cache_database_id(mr.id) end end # Creates the merge request and returns its ID. # # This method will return `nil` if the merge request could not be - # created. + # created, otherwise it will return an Array containing the following + # values: + # + # 1. A MergeRequest instance. + # 2. A boolean indicating if the MR already exists. def create_merge_request author_id, author_found = user_finder.author_id_for(pull_request) description = MarkdownText .format(pull_request.description, pull_request.author, author_found) - # This work must be wrapped in a transaction as otherwise we can leave - # behind incomplete data in the event of an error. This can then lead - # to duplicate key errors when jobs are retried. - MergeRequest.transaction do - attributes = { - iid: pull_request.iid, - title: pull_request.truncated_title, - description: description, - source_project_id: project.id, - target_project_id: project.id, - source_branch: pull_request.formatted_source_branch, - target_branch: pull_request.target_branch, - state: pull_request.state, - milestone_id: milestone_finder.id_for(pull_request), - author_id: author_id, - assignee_id: user_finder.assignee_id_for(pull_request), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - # When creating merge requests there are a lot of hooks that may - # run, for many different reasons. Many of these hooks (e.g. the - # ones used for rendering Markdown) are completely unnecessary and - # may even lead to transaction timeouts. - # - # To ensure importing pull requests has a minimal impact and can - # complete in a reasonable time we bypass all the hooks by inserting - # the row and then retrieving it. We then only perform the - # additional work that is strictly necessary. - merge_request_id = GithubImport - .insert_and_return_id(attributes, project.merge_requests) + attributes = { + iid: pull_request.iid, + title: pull_request.truncated_title, + description: description, + source_project_id: project.id, + target_project_id: project.id, + source_branch: pull_request.formatted_source_branch, + target_branch: pull_request.target_branch, + state: pull_request.state, + milestone_id: milestone_finder.id_for(pull_request), + author_id: author_id, + assignee_id: user_finder.assignee_id_for(pull_request), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } - merge_request = project.merge_requests.find(merge_request_id) - - # These fields are set so we can create the correct merge request - # diffs. - merge_request.source_branch_sha = pull_request.source_branch_sha - merge_request.target_branch_sha = pull_request.target_branch_sha - - merge_request.keep_around_commit - merge_request.merge_request_diffs.create + create_merge_request_without_hooks(project, attributes, pull_request.iid) + end - merge_request.id - end - rescue ActiveRecord::InvalidForeignKey - # It's possible the project has been deleted since scheduling this - # job. In this case we'll just skip creating the merge request. + def insert_git_data(merge_request, already_exists) + insert_or_replace_git_data(merge_request, pull_request.source_branch_sha, pull_request.target_branch_sha, already_exists) end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index e70361c163b..a52866c4b08 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -43,7 +43,7 @@ module Gitlab Rails.logger .info("GitHub importer finished updating repository for #{pname}") - repository_updates_counter.increment(project: pname) + repository_updates_counter.increment end def update_repository?(pr) diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 100f459fdcc..0e7c9ee0d00 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -10,11 +10,13 @@ module Gitlab # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` + # rubocop: disable CodeReuse/ActiveRecord def initialize(project, client) @project = project @client = client @existing_tags = project.releases.pluck(:tag).to_set end + # rubocop: enable CodeReuse/ActiveRecord def execute bulk_insert(Release, build_releases) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 01168abde6c..bc3ea9e9226 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -14,11 +14,13 @@ module Gitlab end # Returns true if we should import the wiki for the project. + # rubocop: disable CodeReuse/ActiveRecord def import_wiki? client.repository(project.import_source)&.has_wiki && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end + # rubocop: enable CodeReuse/ActiveRecord # Imports the repository data. # @@ -78,7 +80,7 @@ module Gitlab end def fail_import(message) - project.mark_import_as_failed(message) + project.import_state.mark_as_failed(message) false end end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index 9be071141db..d2479a8f565 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -18,6 +18,7 @@ module Gitlab Caching.read_integer(cache_key_for(name)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .labels @@ -28,6 +29,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(name) CACHE_KEY % { project: project.id, name: name } diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index 208d15dc144..5625730e796 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -21,6 +21,7 @@ module Gitlab Caching.read_integer(cache_key_for(issuable.milestone_number)) end + # rubocop: disable CodeReuse/ActiveRecord def build_cache mapping = @project .milestones @@ -31,6 +32,7 @@ module Gitlab Caching.write_multiple(mapping) end + # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(iid) CACHE_KEY % { project: project.id, iid: iid } diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index b02b123c98e..9d81441d96e 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -15,6 +15,15 @@ module Gitlab true end + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore + # the visibility of prepended modules. See + # https://github.com/rspec/rspec-mocks/issues/1231 for more details. + if Rails.env.test? + def self.requires_ci_cd_setup? + raise NotImplementedError + end + end + def initialize(project) @project = project end @@ -32,8 +41,7 @@ module Gitlab Gitlab::SidekiqStatus .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) - project.ensure_import_state - project.import_state&.update_column(:jid, jid) + project.import_state.update_column(:jid, jid) Stage::ImportRepositoryWorker .perform_async(project.id) diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb index c3405759631..d2438ee8094 100644 --- a/lib/gitlab/github_import/representation/expose_attribute.rb +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -6,7 +6,7 @@ module Gitlab module ExposeAttribute extend ActiveSupport::Concern - module ClassMethods + class_methods do # Defines getter methods for the given attribute names. # # Example: diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb new file mode 100644 index 00000000000..a4606173f49 --- /dev/null +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class LfsObject + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :oid, :link, :size + + # Builds a lfs_object + def self.from_api_response(lfs_object) + new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size }) + end + + # Builds a new lfs_object using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the raw lfs_object details. The keys of this + # Hash must be Symbols. + def initialize(attributes) + @attributes = attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 4f7324536a0..6a181caf65d 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -19,7 +19,8 @@ module Gitlab Importer::PullRequestsImporter, Importer::IssuesImporter, Importer::DiffNotesImporter, - Importer::NotesImporter + Importer::NotesImporter, + Importer::LfsObjectsImporter ].freeze # project - The project to import the data into. @@ -41,8 +42,6 @@ module Gitlab klass.new(project, client, parallel: false).execute end - project.repository.after_import - true end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index be1259662a7..30283f147ef 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -136,13 +136,17 @@ module Gitlab Caching.write(ID_FOR_EMAIL_CACHE_KEY % email, gitlab_id) end + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) User.for_github_id(id).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_email(email) User.by_any_email(email).pluck(:id).first end + # rubocop: enable CodeReuse/ActiveRecord # Reads an ID from the cache. # diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb index 22719e9a003..86474159f8b 100644 --- a/lib/gitlab/gitlab_import/client.rb +++ b/lib/gitlab/gitlab_import/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitlabImport class Client @@ -32,15 +34,15 @@ module Gitlab api.get("/api/v4/user").parsed end - def issues(project_identifier) - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{PER_PAGE}&page=#{page}").parsed + def issues(project_identifier, **kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed end end - def issue_comments(project_identifier, issue_id) - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{PER_PAGE}&page=#{page}").parsed + def issue_comments(project_identifier, issue_id, **kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed end end @@ -48,23 +50,27 @@ module Gitlab api.get("/api/v4/projects/#{id}").parsed end - def projects - lazy_page_iterator(PER_PAGE) do |page| - api.get("/api/v4/projects?per_page=#{PER_PAGE}&page=#{page}").parsed + def projects(**kwargs) + lazy_page_iterator(**kwargs) do |page, per_page| + api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed end end private - def lazy_page_iterator(per_page) + def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE) Enumerator.new do |y| - page = 1 + page = starting_page + page_limit = (starting_page - 1) + page_limit if page_limit + loop do - items = yield(page) + items = yield(page, per_page) + items.each do |item| y << item end - break if items.empty? || items.size < per_page + + break if items.empty? || items.size < per_page || (page_limit && page >= page_limit) page += 1 end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 195672f5a12..e84863deba8 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitlabImport class Importer @@ -22,22 +24,22 @@ module Gitlab issues = client.issues(project_identifier) issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"]) - body += issue["description"] + body = [@formatter.author_line(issue["author"]["name"])] + body << issue["description"] comments = client.issue_comments(project_identifier, issue["iid"]) if comments.any? - body += @formatter.comments_header + body << @formatter.comments_header end comments.each do |comment| - body += @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) + body << @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) end project.issues.create!( iid: issue["iid"], - description: body, + description: body.join, title: issue["title"], state: issue["state"], updated_at: issue["updated_at"], @@ -52,10 +54,12 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 430b8c10058..35feea17351 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GitlabImport class ProjectCreator diff --git a/lib/gitlab/gl_id.rb b/lib/gitlab/gl_id.rb index a53d156b41f..1ed842c2264 100644 --- a/lib/gitlab/gl_id.rb +++ b/lib/gitlab/gl_id.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GlId def self.gl_id(user) diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 07c0abcce23..435b74806e7 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -1,9 +1,12 @@ +# frozen_string_literal: true + module Gitlab module GlRepository def self.gl_repository(project, is_wiki) "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" end + # rubocop: disable CodeReuse/ActiveRecord def self.parse(gl_repository) match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) unless match_data @@ -16,5 +19,6 @@ module Gitlab [project, wiki] end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index deaa14c8434..9b1794eec91 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable Metrics/AbcSize module Gitlab @@ -6,7 +8,7 @@ module Gitlab def add_gon_variables gon.api_version = 'v4' - gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_avatar_url = default_avatar_url gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path @@ -30,5 +32,30 @@ module Gitlab gon.current_user_avatar_url = current_user.avatar_url end end + + # Exposes the state of a feature flag to the frontend code. + # + # name - The name of the feature flag, e.g. `my_feature`. + # args - Any additional arguments to pass to `Feature.enabled?`. This allows + # you to check if a flag is enabled for a particular user. + def push_frontend_feature_flag(name, *args) + var_name = name.to_s.camelize(:lower) + enabled = Feature.enabled?(name, *args) + + # Here the `true` argument signals gon that the value should be merged + # into any existing ones, instead of overwriting them. This allows you to + # use this method to push multiple feature flags. + gon.push({ features: { var_name => enabled } }, true) + end + + def default_avatar_url + # We can't use ActionController::Base.helpers.image_url because it + # doesn't return an actual URL because request is nil for some reason. + # + # We also can't use Gitlab::Utils.append_path because the image path + # may be an absolute URL. + URI.join(Gitlab.config.gitlab.url, + ActionController::Base.helpers.image_path('no_avatar.png')).to_s + end end end diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index b1dbf554e41..52d714880b5 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GoogleCodeImport class Client diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 46b49128140..1e7203cb82a 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GoogleCodeImport class Importer @@ -78,6 +80,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues return unless repo.issues @@ -101,7 +104,7 @@ module Gitlab if username.start_with?("@") username = username[1..-1] - if user = User.find_by(username: username) + if user = UserFinder.new(username).find_by_username assignee_id = user.id end end @@ -123,6 +126,7 @@ module Gitlab import_issue_comments(issue, comments) end end + # rubocop: enable CodeReuse/ActiveRecord def import_issue_labels(raw_issue) labels = [] @@ -200,27 +204,27 @@ module Gitlab "Status: #{name}" end - def linkify_issues(s) - s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - s = s.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') - s + def linkify_issues(str) + str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + str = str.gsub(/([Cc]omment) #([0-9]+)/, '\1 \2') + str end - def escape_for_markdown(s) + def escape_for_markdown(str) # No headings and lists - s = s.gsub(/^#/, "\\#") - s = s.gsub(/^-/, "\\-") + str = str.gsub(/^#/, "\\#") + str = str.gsub(/^-/, "\\-") # No inline code - s = s.gsub("`", "\\`") + str = str.gsub("`", "\\`") # Carriage returns make me sad - s = s.delete("\r") + str = str.delete("\r") # Markdown ignores single newlines, but we need them as <br />. - s = s.gsub("\n", " \n") + str = str.gsub("\n", " \n") - s + str end def create_label(name) diff --git a/lib/gitlab/google_code_import/project_creator.rb b/lib/gitlab/google_code_import/project_creator.rb index 326cfcaa8af..eaef85acb98 100644 --- a/lib/gitlab/google_code_import/project_creator.rb +++ b/lib/gitlab/google_code_import/project_creator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GoogleCodeImport class ProjectCreator diff --git a/lib/gitlab/google_code_import/repository.rb b/lib/gitlab/google_code_import/repository.rb index ad33fc2cad2..19627c8cd35 100644 --- a/lib/gitlab/google_code_import/repository.rb +++ b/lib/gitlab/google_code_import/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module GoogleCodeImport class Repository diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 413872d7e08..32f61b1d65c 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Gpg extend self @@ -54,7 +56,11 @@ module Gitlab fingerprints = CurrentKeyChain.fingerprints_from_key(key) GPGME::Key.find(:public, fingerprints).flat_map do |raw_key| - raw_key.uids.map { |uid| { name: uid.name, email: uid.email.downcase } } + raw_key.uids.each_with_object([]) do |uid, arr| + name = uid.name.force_encoding('UTF-8') + email = uid.email.force_encoding('UTF-8') + arr << { name: name, email: email.downcase } if name.valid_encoding? && email.valid_encoding? + end end end end @@ -67,8 +73,10 @@ module Gitlab if MUTEX.locked? && MUTEX.owned? optimistic_using_tmp_keychain(&block) else - MUTEX.synchronize do - optimistic_using_tmp_keychain(&block) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + MUTEX.synchronize do + optimistic_using_tmp_keychain(&block) + end end end end diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index 6d2278d0876..4fbb87385c3 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Gpg class Commit @@ -26,6 +28,7 @@ module Gitlab !!(signature_text && signed_text) end + # rubocop: disable CodeReuse/ActiveRecord def signature return unless has_signature? @@ -36,13 +39,13 @@ module Gitlab @signature = create_cached_signature! end + # rubocop: enable CodeReuse/ActiveRecord def update_signature!(cached_signature) using_keychain do |gpg_key| - cached_signature.update_attributes!(attributes(gpg_key)) + cached_signature.update!(attributes(gpg_key)) + @signature = cached_signature end - - @signature = cached_signature end private @@ -55,11 +58,15 @@ module Gitlab # the proper signature. # NOTE: the invoked method is #fingerprint but it's only returning # 16 characters (the format used by keyid) instead of 40. - gpg_key = find_gpg_key(verified_signature.fingerprint) + fingerprint = verified_signature&.fingerprint + + break unless fingerprint + + gpg_key = find_gpg_key(fingerprint) if gpg_key Gitlab::Gpg::CurrentKeyChain.add(gpg_key.key) - @verified_signature = nil + clear_memoization(:verified_signature) end yield gpg_key @@ -67,9 +74,16 @@ module Gitlab end def verified_signature - @verified_signature ||= GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| + strong_memoize(:verified_signature) { gpgme_signature } + end + + def gpgme_signature + GPGME::Crypto.new.verify(signature_text, signed_text: signed_text) do |verified_signature| + # Return the first signature for now: https://gitlab.com/gitlab-org/gitlab-ce/issues/54932 break verified_signature end + rescue GPGME::Error + nil end def create_cached_signature! @@ -88,7 +102,7 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, gpg_key: gpg_key, - gpg_key_primary_keyid: gpg_key&.keyid || verified_signature.fingerprint, + gpg_key_primary_keyid: gpg_key&.keyid || verified_signature&.fingerprint, gpg_key_user_name: user_infos[:name], gpg_key_user_email: user_infos[:email], verification_status: verification_status @@ -98,7 +112,7 @@ module Gitlab def verification_status(gpg_key) return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? - return :unverified unless verified_signature.valid? + return :unverified unless verified_signature&.valid? if gpg_key.verified_and_belongs_to_email?(@commit.committer_email) :verified @@ -113,9 +127,11 @@ module Gitlab gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end + # rubocop: disable CodeReuse/ActiveRecord def find_gpg_key(keyid) GpgKey.find_by(primary_keyid: keyid) || GpgKeySubkey.find_by(keyid: keyid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb index 1991911ef6a..d892d27a917 100644 --- a/lib/gitlab/gpg/invalid_gpg_signature_updater.rb +++ b/lib/gitlab/gpg/invalid_gpg_signature_updater.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Gpg class InvalidGpgSignatureUpdater @@ -5,6 +7,7 @@ module Gitlab @gpg_key = gpg_key end + # rubocop: disable CodeReuse/ActiveRecord def run GpgSignature .select(:id, :commit_sha, :project_id) @@ -12,6 +15,7 @@ module Gitlab .where(gpg_key_primary_keyid: @gpg_key.keyids) .find_each { |sig| sig.gpg_commit&.update_signature!(sig) } end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 1e1fdabca93..9bb1e8fc7a2 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -1,9 +1,15 @@ +# frozen_string_literal: true + module Gitlab module GrapeLogging module Formatters class LogrageWithTimestamp + include Gitlab::EncodingHelper + def call(severity, datetime, _, data) time = data.delete :time + data[:params] = process_params(data) + attributes = { time: datetime.utc.iso8601(3), severity: severity, @@ -13,6 +19,27 @@ module Gitlab }.merge(data) ::Lograge.formatter.call(attributes) + "\n" end + + private + + def process_params(data) + return [] unless data.has_key?(:params) + + data[:params] + .each_pair + .map { |k, v| { key: k, value: utf8_encode_values(v) } } + end + + def utf8_encode_values(data) + case data + when Hash + data.merge(data) { |k, v| utf8_encode_values(v) } + when Array + data.map { |v| utf8_encode_values(v) } + when String + encode_utf8(data) + end + end end end end diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb new file mode 100644 index 00000000000..fa4c5d86d44 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional correlation id the grape logger +module Gitlab + module GrapeLogging + module Loggers + class CorrelationIdLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb new file mode 100644 index 00000000000..e3b9c59bd6e --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This module adds additional performance metrics to the grape logger +module Gitlab + module GrapeLogging + module Loggers + class PerfLogger < ::GrapeLogging::Loggers::Base + def parameters(_, _) + { gitaly_calls: Gitlab::GitalyClient.get_request_count } + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb index 0adac79f25a..705e23adff2 100644 --- a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This grape_logging module (https://github.com/aserafin/grape_logging) makes it # possible to log how much time an API request was queued by Workhorse. module Gitlab diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb new file mode 100644 index 00000000000..f3146b4dfd9 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/route_logger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the details of the action +module Gitlab + module GrapeLogging + module Loggers + class RouteLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + endpoint = request.env[Grape::Env::API_ENDPOINT] + route = endpoint&.route&.pattern&.origin + + return {} unless route + + { route: route } + rescue + # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + {} + end + end + end + end +end diff --git a/lib/gitlab/grape_logging/loggers/user_logger.rb b/lib/gitlab/grape_logging/loggers/user_logger.rb index fa172861967..6caa6c715e7 100644 --- a/lib/gitlab/grape_logging/loggers/user_logger.rb +++ b/lib/gitlab/grape_logging/loggers/user_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This grape_logging module (https://github.com/aserafin/grape_logging) makes it # possible to log the user who performed the Grape API action by retrieving # the user context from the request environment. diff --git a/lib/gitlab/graphql.rb b/lib/gitlab/graphql.rb new file mode 100644 index 00000000000..8a59e83974f --- /dev/null +++ b/lib/gitlab/graphql.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + StandardGraphqlError = Class.new(StandardError) + + def self.enabled? + Feature.enabled?(:graphql, default_enabled: true) + end + end +end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb new file mode 100644 index 00000000000..5e48bf9043d --- /dev/null +++ b/lib/gitlab/graphql/authorize.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + # Allow fields to declare permissions their objects must have. The field + # will be set to nil unless all required permissions are present. + module Authorize + extend ActiveSupport::Concern + + def self.use(schema_definition) + schema_definition.instrument(:field, Instrumentation.new) + end + + def required_permissions + # If the `#authorize` call is used on multiple classes, we add the + # permissions specified on a subclass, to the ones that were specified + # on it's superclass. + @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions) + superclass.required_permissions.dup + else + [] + end + end + + def authorize(*permissions) + required_permissions.concat(permissions) + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb new file mode 100644 index 00000000000..a56c4f6368d --- /dev/null +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + module AuthorizeResource + extend ActiveSupport::Concern + + included do + extend Gitlab::Graphql::Authorize + end + + def find_object(*args) + raise NotImplementedError, "Implement #find_object in #{self.class.name}" + end + + def authorized_find(*args) + object = find_object(*args) + + object if authorized?(object) + end + + def authorized_find!(*args) + object = find_object(*args) + authorize!(object) + + object + end + + def authorize!(object) + unless authorized?(object) + raise Gitlab::Graphql::Errors::ResourceNotAvailable, + "The resource that you are attempting to access does not exist or you don't have permission to perform this action" + end + end + + def authorized?(object) + self.class.required_permissions.all? do |ability| + # The actions could be performed across multiple objects. In which + # case the current user is common, and we could benefit from the + # caching in `DeclarativePolicy`. + Ability.allowed?(current_user, ability, object, scope: :user) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb new file mode 100644 index 00000000000..d638d2b43ee --- /dev/null +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + class Instrumentation + # Replace the resolver for the field with one that will only return the + # resolved object if the permissions check is successful. + # + # Collections are not supported. Apply permissions checks for those at the + # database level instead, to avoid loading superfluous data from the DB + def instrument(_type, field) + field_definition = field.metadata[:type_class] + return field unless field_definition.respond_to?(:required_permissions) + return field if field_definition.required_permissions.empty? + + old_resolver = field.resolve_proc + + new_resolver = -> (obj, args, ctx) do + resolved_obj = old_resolver.call(obj, args, ctx) + checker = build_checker(ctx[:current_user], field_definition.required_permissions) + + if resolved_obj.respond_to?(:then) + resolved_obj.then(&checker) + else + checker.call(resolved_obj) + end + end + + field.redefine do + resolve(new_resolver) + end + end + + private + + def build_checker(current_user, abilities) + proc do |obj| + # Load the elements if they weren't loaded by BatchLoader yet + obj = obj.sync if obj.respond_to?(:sync) + obj if abilities.all? { |ability| Ability.allowed?(current_user, ability, obj) } + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb new file mode 100644 index 00000000000..fbccdfa7b08 --- /dev/null +++ b/lib/gitlab/graphql/connections.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + def self.use(_schema) + GraphQL::Relay::BaseConnection.register_connection_implementation( + ActiveRecord::Relation, + Gitlab::Graphql::Connections::KeysetConnection + ) + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb new file mode 100644 index 00000000000..851054c0393 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Connections + class KeysetConnection < GraphQL::Relay::BaseConnection + def cursor_from_node(node) + encode(node[order_field].to_s) + end + + # rubocop: disable CodeReuse/ActiveRecord + def sliced_nodes + @sliced_nodes ||= + begin + sliced = nodes + + sliced = sliced.where(before_slice) if before.present? + sliced = sliced.where(after_slice) if after.present? + + sliced + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def paged_nodes + if first && last + raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + end + + if last + sliced_nodes.last(limit_value) + else + sliced_nodes.limit(limit_value) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def before_slice + if sort_direction == :asc + table[order_field].lt(decode(before)) + else + table[order_field].gt(decode(before)) + end + end + + def after_slice + if sort_direction == :asc + table[order_field].gt(decode(after)) + else + table[order_field].lt(decode(after)) + end + end + + def limit_value + @limit_value ||= [first, last, max_page_size].compact.min + end + + def table + nodes.arel_table + end + + def order_info + @order_info ||= nodes.order_values.first + end + + def order_field + @order_field ||= order_info&.expr&.name || nodes.primary_key + end + + def sort_direction + @order_direction ||= order_info&.direction || :desc + end + end + end + end +end diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb new file mode 100644 index 00000000000..fe74549e322 --- /dev/null +++ b/lib/gitlab/graphql/errors.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Errors + BaseError = Class.new(GraphQL::ExecutionError) + ArgumentError = Class.new(BaseError) + ResourceNotAvailable = Class.new(BaseError) + end + end +end diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb new file mode 100644 index 00000000000..365b7cca24f --- /dev/null +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module ExposePermissions + extend ActiveSupport::Concern + prepended do + def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + field :user_permissions, permission_type, + description: description, + null: false, + resolve: -> (obj, _, _) { obj } + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb new file mode 100644 index 00000000000..5a0099dc6b1 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchModelLoader + attr_reader :model_class, :model_id + + def initialize(model_class, model_id) + @model_class, @model_id = model_class, model_id + end + + # rubocop: disable CodeReuse/ActiveRecord + def find + BatchLoader.for({ model: model_class, id: model_id }).batch do |loader_info, loader| + per_model = loader_info.group_by { |info| info[:model] } + per_model.each do |model, info| + ids = info.map { |i| i[:id] } + results = model.where(id: ids) + + results.each { |record| loader.call({ model: model, id: record.id }, record) } + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/graphql/mount_mutation.rb b/lib/gitlab/graphql/mount_mutation.rb new file mode 100644 index 00000000000..9048967d4e1 --- /dev/null +++ b/lib/gitlab/graphql/mount_mutation.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module MountMutation + extend ActiveSupport::Concern + + class_methods do + def mount_mutation(mutation_class) + # Using an underscored field name symbol will make `graphql-ruby` + # standardize the field name + field mutation_class.graphql_name.underscore.to_sym, + mutation: mutation_class + end + end + end + end +end diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb new file mode 100644 index 00000000000..7f69bf601d6 --- /dev/null +++ b/lib/gitlab/graphql/present.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Present + extend ActiveSupport::Concern + prepended do + def self.present_using(kls) + @presenter_class = kls + end + + def self.presenter_class + @presenter_class + end + end + + def self.use(schema_definition) + schema_definition.instrument(:field, Instrumentation.new) + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb new file mode 100644 index 00000000000..ab03c40c22d --- /dev/null +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Present + class Instrumentation + def instrument(type, field) + return field unless field.metadata[:type_class] + + presented_in = field.metadata[:type_class].owner + return field unless presented_in.respond_to?(:presenter_class) + return field unless presented_in.presenter_class + + old_resolver = field.resolve_proc + + resolve_with_presenter = -> (presented_type, args, context) do + # We need to wrap the original presentation type into a type that + # uses the presenter as an object. + object = presented_type.object + + if object.is_a?(presented_in.presenter_class) + next old_resolver.call(presented_type, args, context) + end + + presenter = presented_in.presenter_class.new(object, **context.to_h) + wrapped = presented_type.class.new(presenter, context) + + old_resolver.call(wrapped, args, context) + end + + field.redefine do + resolve(resolve_with_presenter) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb new file mode 100644 index 00000000000..b13ea37c21f --- /dev/null +++ b/lib/gitlab/graphql/variables.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class Variables + Invalid = Class.new(Gitlab::Graphql::StandardGraphqlError) + + def initialize(param) + @param = param + end + + def to_h + ensure_hash(@param) + end + + private + + # Handle form data, JSON body, or a blank value + def ensure_hash(ambiguous_param) + case ambiguous_param + when String + if ambiguous_param.present? + ensure_hash(JSON.parse(ambiguous_param)) + else + {} + end + when Hash, ActionController::Parameters + ambiguous_param + when nil + {} + else + raise Invalid, "Unexpected parameter: #{ambiguous_param}" + end + rescue JSON::ParserError => e + raise Invalid.new(e) + end + end + end +end diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 3caf9036459..66e1b2e78b4 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Graphs class Commits @@ -18,7 +20,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= @commits.size / (@duration + 1) + @commit_per_day ||= (@commits.size.to_f / (@duration + 1)).round(1) end def collect_data diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb new file mode 100644 index 00000000000..bf463077dcc --- /dev/null +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module HashedStorage + # Hashed Storage Migrator + # + # This is responsible for scheduling and flagging projects + # to be migrated from Legacy to Hashed storage, either one by one or in bulk. + class Migrator + BATCH_SIZE = 100 + + # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + def bulk_schedule(start:, finish:) + ::HashedStorage::MigratorWorker.perform_async(start, finish) + end + + # Start migration of projects from specified range + # + # Flagging a project to be migrated is a synchronous action + # but the migration runs through async jobs + # + # @param [Integer] start first project id for the range + # @param [Integer] finish last project id for the range + # rubocop: disable CodeReuse/ActiveRecord + def bulk_migrate(start:, finish:) + projects = build_relation(start, finish) + + projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| + migrate(project) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # Flag a project to be migrated to Hashed Storage + # + # @param [Project] project that will be migrated + def migrate(project) + Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." + + project.migrate_to_hashed_storage! + rescue => err + Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") + end + + def rollback(project) + # TODO: implement rollback strategy + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def build_relation(start, finish) + relation = Project + table = Project.arel_table + + relation = relation.where(table[:id].gteq(start)) if start + relation = relation.where(table[:id].lteq(finish)) if finish + + relation + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 8aba42ccfce..38f552fab03 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HashedStorage module RakeHelper @@ -9,27 +11,45 @@ module Gitlab ENV.fetch('LIMIT', 500).to_i end + def self.range_from + ENV['ID_FROM'] + end + + def self.range_to + ENV['ID_TO'] + end + + def self.range_single_item? + !range_from.nil? && range_from == range_to + end + + # rubocop: disable CodeReuse/ActiveRecord def self.project_id_batches(&block) - Project.with_unmigrated_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches ids = relation.pluck(:id) yield ids.min, ids.max end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.legacy_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version < :version OR projects.storage_version IS NULL', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def self.hashed_attachments_relation Upload.joins(<<~SQL).where('projects.storage_version >= :version', version: Project::HASHED_STORAGE_FEATURES[:attachments]) JOIN projects ON (uploads.model_type='Project' AND uploads.model_id=projects.id) SQL end + # rubocop: enable CodeReuse/ActiveRecord def self.relation_summary(relation_name, relation) relation_count = relation.count @@ -50,6 +70,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def self.listing(relation_name, relation) relation_count = relation_summary(relation_name, relation) return unless relation_count > 0 @@ -66,6 +87,7 @@ module Gitlab break if index + 1 >= limit end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 8b365dab185..1d31f59999c 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module BaseAbstractCheck diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index e27e16ddaf6..2bcd25cd3cc 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks class DbCheck @@ -17,7 +19,7 @@ module Gitlab def check catch_timeout 10.seconds do if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s else ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb deleted file mode 100644 index fcbf266b80b..00000000000 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ /dev/null @@ -1,168 +0,0 @@ -module Gitlab - module HealthChecks - class FsShardsCheck - extend BaseAbstractCheck - RANDOM_STRING = SecureRandom.hex(1000).freeze - COMMAND_TIMEOUT = '1'.freeze - TIMEOUT_EXECUTABLE = 'timeout'.freeze - - class << self - def readiness - repository_storages.map do |storage_name| - begin - if !storage_circuitbreaker_test(storage_name) - HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) - elsif !storage_stat_test(storage_name) - HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - else - with_temp_file(storage_name) do |tmp_file_path| - if !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) - else - HealthChecks::Result.new(true, nil, shard: storage_name) - end - end - end - rescue RuntimeError => ex - message = "unexpected error #{ex} when checking storage #{storage_name}" - Rails.logger.error(message) - HealthChecks::Result.new(false, message, shard: storage_name) - end - end - end - - def metrics - repository_storages.flat_map do |storage_name| - [ - storage_stat_metrics(storage_name), - storage_write_metrics(storage_name), - storage_read_metrics(storage_name), - storage_circuitbreaker_metrics(storage_name) - ].flatten - end - end - - private - - def operation_metrics(ok_metric, latency_metric, **labels) - result, elapsed = yield - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - rescue RuntimeError => ex - Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") - [metric(ok_metric, 0, **labels)] - end - - def repository_storages - storages_paths.keys - end - - def storages_paths - Gitlab.config.repositories.storages - end - - def exec_with_timeout(cmd_args, *args, &block) - Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) - end - - def with_temp_file(storage_name) - temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } - yield temp_file_path - ensure - delete_test_file(temp_file_path) - end - - def storage_path(storage_name) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - storages_paths[storage_name]&.legacy_disk_path - end - end - - # All below test methods use shell commands to perform actions on storage volumes. - # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, - # we can rely on shell commands to be terminated once `timeout` kills them. - # - # However we also fallback to pure Ruby file operations in case a specific shell command is missing - # so we are still able to perform healthchecks and gather metrics from such system. - - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status.zero? - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT - end - - def storage_stat_test(storage_name) - stat_path = File.join(storage_path(storage_name), '.') - begin - _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status.zero? - rescue Errno::ENOENT - File.exist?(stat_path) && File::Stat.new(stat_path).readable? - end - end - - def storage_write_test(tmp_path) - _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT - written_bytes == RANDOM_STRING.length - end - - def storage_read_test(tmp_path) - _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - file_contents = File.read(tmp_path) rescue Errno::ENOENT - file_contents == RANDOM_STRING - end - - def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } - rescue Gitlab::Git::Storage::Inaccessible - nil - end - - def storage_stat_metrics(storage_name) - operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do - with_timing { storage_stat_test(storage_name) } - end - end - - def storage_write_metrics(storage_name) - operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - with_timing { storage_write_test(tmp_file_path) } - end - end - end - - def storage_read_metrics(storage_name) - operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - storage_write_test(tmp_file_path) # writes data used by read test - with_timing { storage_read_test(tmp_file_path) } - end - end - end - - def storage_circuitbreaker_metrics(storage_name) - operation_metrics(:filesystem_circuitbreaker, - :filesystem_circuitbreaker_latency_seconds, - shard: storage_name) do - with_timing { storage_circuitbreaker_test(storage_name) } - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index 11416c002e3..898733fea5d 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks class GitalyCheck @@ -13,14 +15,14 @@ module Gitlab end def metrics - repository_storages.flat_map do |storage_name| - result, elapsed = with_timing { check(storage_name) } - labels = { shard: storage_name } + Gitaly::Server.all.flat_map do |server| + result, elapsed = with_timing { server.read_writeable? } + labels = { shard: server.storage } [ - metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_success", result ? 1 : 0, **labels), metric("#{metric_prefix}_latency_seconds", elapsed, **labels) - ].flatten + ] end end @@ -36,10 +38,6 @@ module Gitlab METRIC_PREFIX end - def successful?(result) - result[:success] - end - def repository_storages storages.keys end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index d62d9136886..62a5216d159 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,3 +1,6 @@ -module Gitlab::HealthChecks # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab::HealthChecks Metric = Struct.new(:name, :value, :labels) end diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb index b3c759b4730..2a8f9d31cd5 100644 --- a/lib/gitlab/health_checks/prometheus_text_format.rb +++ b/lib/gitlab/health_checks/prometheus_text_format.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks class PrometheusTextFormat diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb index 0eb9b77634a..0c8fe83893b 100644 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ b/lib/gitlab/health_checks/redis/cache_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module Redis @@ -19,11 +21,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Cache.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb index f322fe831b8..b1e33b9f459 100644 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ b/lib/gitlab/health_checks/redis/queues_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module Redis @@ -19,11 +21,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::Queues.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index 8ceb0a0aa46..f7e46fce134 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module Redis diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb index 07e6f707998..285ac271929 100644 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ b/lib/gitlab/health_checks/redis/shared_state_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module Redis @@ -19,11 +21,13 @@ module Gitlab result == 'PONG' end + # rubocop: disable CodeReuse/ActiveRecord def check catch_timeout 10.seconds do Gitlab::Redis::SharedState.with(&:ping) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index e323e2c9723..d32a6980eb8 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,3 +1,6 @@ -module Gitlab::HealthChecks # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab::HealthChecks Result = Struct.new(:success, :message, :labels) end diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index 96945ce5b20..3588260d6eb 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module HealthChecks module SimpleAbstractCheck diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 5408a1a6838..a4e60bbd828 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -1,20 +1,28 @@ +# frozen_string_literal: true + module Gitlab class Highlight - def self.highlight(blob_name, blob_content, repository: nil, plain: false) - new(blob_name, blob_content, repository: repository) + TIMEOUT_BACKGROUND = 30.seconds + TIMEOUT_FOREGROUND = 3.seconds + MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte + + def self.highlight(blob_name, blob_content, language: nil, plain: false) + new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) end attr_reader :blob_name - def initialize(blob_name, blob_content, repository: nil) + def initialize(blob_name, blob_content, language: nil) @formatter = Rouge::Formatters::HTMLGitlab - @repository = repository + @language = language @blob_name = blob_name @blob_content = blob_content end def highlight(text, continue: true, plain: false) + plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE + highlighted_text = highlight_text(text, continue: continue, plain: plain) highlighted_text = link_dependencies(text, highlighted_text) if blob_name highlighted_text @@ -31,11 +39,9 @@ module Gitlab private def custom_language - language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') - - return nil unless language_name + return nil unless @language - Rouge::Lexer.find_fancy(language_name) + Rouge::Lexer.find_fancy(@language) end def highlight_text(text, continue: true, plain: false) @@ -51,11 +57,20 @@ module Gitlab end def highlight_rich(text, continue: true) - @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + tag = lexer.tag + tokens = lexer.lex(text, continue: continue) + Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + rescue Timeout::Error => e + Gitlab::Sentry.track_exception(e) + highlight_plain(text) rescue highlight_plain(text) end + def timeout_time + Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + end + def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb new file mode 100644 index 00000000000..d54175bce81 --- /dev/null +++ b/lib/gitlab/hook_data/base_builder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class BaseBuilder + attr_accessor :object + + MARKDOWN_SIMPLE_IMAGE = %r{ + #{::Gitlab::Regex.markdown_code_or_html_blocks} + | + (?<image> + ! + \[(?<title>[^\n]*?)\] + \((?<url>(?!(https?://|//))[^\n]+?)\) + ) + }mx.freeze + + def initialize(object) + @object = object + end + + private + + def absolute_image_urls(markdown_text) + return markdown_text unless markdown_text.present? + + markdown_text.gsub(MARKDOWN_SIMPLE_IMAGE) do + if $~[:image] + url = $~[:url] + url = "#{uploads_prefix}#{url}" if url.start_with?('/uploads') + url = "/#{url}" unless url.start_with?('/') + + "![#{$~[:title]}](#{Gitlab.config.gitlab.url}#{url})" + else + $~[0] + end + end + end + + def uploads_prefix + project&.full_path || '' + end + + def project + return unless object.respond_to?(:project) + + object.project + end + end + end +end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 6ab36676127..0803df65632 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -1,13 +1,11 @@ +# frozen_string_literal: true + module Gitlab module HookData - class IssuableBuilder + class IssuableBuilder < BaseBuilder CHANGES_KEYS = %i[previous current].freeze - attr_accessor :issuable - - def initialize(issuable) - @issuable = issuable - end + alias_method :issuable, :object def build(user: nil, changes: {}) hook_data = { @@ -32,7 +30,7 @@ module Gitlab end def safe_keys - issuable_builder::SAFE_HOOK_ATTRIBUTES + issuable_builder::SAFE_HOOK_RELATIONS + issuable_builder.safe_hook_attributes + issuable_builder::SAFE_HOOK_RELATIONS end private diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index f9b1a3caf5e..c99353b9d49 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -1,53 +1,54 @@ +# frozen_string_literal: true + module Gitlab module HookData - class IssueBuilder - SAFE_HOOK_ATTRIBUTES = %i[ - assignee_id - author_id - closed_at - confidential - created_at - description - due_date - id - iid - last_edited_at - last_edited_by_id - milestone_id - moved_to_id - project_id - relative_position - state - time_estimate - title - updated_at - updated_by_id - ].freeze - + class IssueBuilder < BaseBuilder SAFE_HOOK_RELATIONS = %i[ assignees labels total_time_spent ].freeze - attr_accessor :issue - - def initialize(issue) - @issue = issue + def self.safe_hook_attributes + %i[ + assignee_id + author_id + closed_at + confidential + created_at + description + due_date + id + iid + last_edited_at + last_edited_by_id + milestone_id + moved_to_id + project_id + relative_position + state + time_estimate + title + updated_at + updated_by_id + ].freeze end + alias_method :issue, :object + def build attrs = { - url: Gitlab::UrlBuilder.build(issue), - total_time_spent: issue.total_time_spent, - human_total_time_spent: issue.human_total_time_spent, - human_time_estimate: issue.human_time_estimate, - assignee_ids: issue.assignee_ids, - assignee_id: issue.assignee_ids.first # This key is deprecated + description: absolute_image_urls(issue.description), + url: Gitlab::UrlBuilder.build(issue), + total_time_spent: issue.total_time_spent, + human_total_time_spent: issue.human_total_time_spent, + human_time_estimate: issue.human_time_estimate, + assignee_ids: issue.assignee_ids, + assignee_id: issue.assignee_ids.first # This key is deprecated } - issue.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) - .merge!(attrs) + issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) + .merge!(attrs) end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index aff786864f2..ad38e26e40a 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -1,33 +1,37 @@ +# frozen_string_literal: true + module Gitlab module HookData - class MergeRequestBuilder - SAFE_HOOK_ATTRIBUTES = %i[ - assignee_id - author_id - created_at - description - head_pipeline_id - id - iid - last_edited_at - last_edited_by_id - merge_commit_sha - merge_error - merge_params - merge_status - merge_user_id - merge_when_pipeline_succeeds - milestone_id - source_branch - source_project_id - state - target_branch - target_project_id - time_estimate - title - updated_at - updated_by_id - ].freeze + class MergeRequestBuilder < BaseBuilder + def self.safe_hook_attributes + %i[ + assignee_id + author_id + created_at + description + head_pipeline_id + id + iid + last_edited_at + last_edited_by_id + merge_commit_sha + merge_error + merge_params + merge_status + merge_user_id + merge_when_pipeline_succeeds + milestone_id + source_branch + source_project_id + state + target_branch + target_project_id + time_estimate + title + updated_at + updated_by_id + ].freeze + end SAFE_HOOK_RELATIONS = %i[ assignee @@ -35,14 +39,11 @@ module Gitlab total_time_spent ].freeze - attr_accessor :merge_request - - def initialize(merge_request) - @merge_request = merge_request - end + alias_method :merge_request, :object def build attrs = { + description: absolute_image_urls(merge_request.description), url: Gitlab::UrlBuilder.build(merge_request), source: merge_request.source_project.try(:hook_attrs), target: merge_request.target_project.hook_attrs, @@ -53,8 +54,8 @@ module Gitlab human_time_estimate: merge_request.human_time_estimate } - merge_request.attributes.with_indifferent_access.slice(*SAFE_HOOK_ATTRIBUTES) - .merge!(attrs) + merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) + .merge!(attrs) end end end diff --git a/lib/gitlab/hook_data/note_builder.rb b/lib/gitlab/hook_data/note_builder.rb new file mode 100644 index 00000000000..ae30ef6364b --- /dev/null +++ b/lib/gitlab/hook_data/note_builder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class NoteBuilder < BaseBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + attachment + author_id + change_position + commit_id + created_at + discussion_id + id + line_code + note + noteable_id + noteable_type + original_position + position + project_id + resolved_at + resolved_by_id + resolved_by_push + st_diff + system + type + updated_at + updated_by_id + ].freeze + + alias_method :note, :object + + def build + note + .attributes + .with_indifferent_access + .slice(*SAFE_HOOK_ATTRIBUTES) + .merge( + description: absolute_image_urls(note.note), + url: Gitlab::UrlBuilder.build(note) + ) + end + end + end +end diff --git a/lib/gitlab/hook_data/wiki_page_builder.rb b/lib/gitlab/hook_data/wiki_page_builder.rb new file mode 100644 index 00000000000..67f06b1ca46 --- /dev/null +++ b/lib/gitlab/hook_data/wiki_page_builder.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class WikiPageBuilder < BaseBuilder + alias_method :wiki_page, :object + + def build + wiki_page + .attributes + .merge( + 'content' => absolute_image_urls(wiki_page.content) + ) + end + end + end +end diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 9aca3b0fb26..bcd9e2be35f 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This class is used as a proxy for all outbounding http connection # coming from callbacks, services and hooks. The direct use of the HTTParty # is discouraged because it can lead to several security problems, like SSRF @@ -5,9 +7,16 @@ module Gitlab class HTTP BlockedUrlError = Class.new(StandardError) + RedirectionTooDeep = Class.new(StandardError) include HTTParty # rubocop:disable Gitlab/HTTParty connection_adapter ProxyHTTPConnectionAdapter + + def self.perform_request(http_method, path, options, &block) + super + rescue HTTParty::RedirectionTooDeep + raise RedirectionTooDeep + end end end diff --git a/lib/gitlab/http_io.rb b/lib/gitlab/http_io.rb new file mode 100644 index 00000000000..6a9fb85b054 --- /dev/null +++ b/lib/gitlab/http_io.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +## +# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) +# source: https://gitlab.com/snippets/1685610 +module Gitlab + class HttpIO + BUFFER_SIZE = 128.kilobytes + + InvalidURLError = Class.new(StandardError) + FailedToGetChunkError = Class.new(StandardError) + + attr_reader :uri, :size + attr_reader :tell + attr_reader :chunk, :chunk_range + + alias_method :pos, :tell + + def initialize(url, size) + raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url) + + @uri = URI(url) + @size = size + @tell = 0 + end + + def close + # no-op + end + + def binmode + # no-op + end + + def binmode? + true + end + + def path + nil + end + + def url + @uri.to_s + end + + def seek(pos, where = IO::SEEK_SET) + new_pos = + case where + when IO::SEEK_END + size + pos + when IO::SEEK_SET + pos + when IO::SEEK_CUR + tell + pos + else + -1 + end + + raise 'new position is outside of file' if new_pos < 0 || new_pos > size + + @tell = new_pos + end + + def eof? + tell == size + end + + def each_line + until eof? + line = readline + break if line.nil? + + yield(line) + end + end + + def read(length = nil, outbuf = nil) + out = [] + + length ||= size - tell + + until length <= 0 || eof? + data = get_chunk + break if data.empty? + + chunk_bytes = [BUFFER_SIZE - chunk_offset, length].min + data_slice = data.byteslice(0, chunk_bytes) + + out << data_slice + @tell += data_slice.bytesize + length -= data_slice.bytesize + end + + out = out.join + + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality + if outbuf + outbuf.replace(out) + end + + out + end + + def readline + out = [] + + until eof? + data = get_chunk + new_line = data.index("\n") + + if !new_line.nil? + out << data[0..new_line] + @tell += new_line + 1 + break + else + out << data + @tell += data.bytesize + end + end + + out.join + end + + def write(data) + raise NotImplementedError + end + + def truncate(offset) + raise NotImplementedError + end + + def flush + raise NotImplementedError + end + + def present? + true + end + + private + + ## + # The below methods are not implemented in IO class + # + def in_range? + @chunk_range&.include?(tell) + end + + def get_chunk + unless in_range? + response = Net::HTTP.start(uri.hostname, uri.port, proxy_from_env: true, use_ssl: uri.scheme == 'https') do |http| + http.request(request) + end + + raise FailedToGetChunkError unless response.code == '200' || response.code == '206' + + @chunk = response.body.force_encoding(Encoding::BINARY) + @chunk_range = response.content_range + + ## + # Note: If provider does not return content_range, then we set it as we requested + # Provider: minio + # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206 + # Provider: AWS + # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206 + # Provider: GCS + # - When the file size is larger than requested Content-range, the Content-range is included in responses with Net::HTTPPartialContent 206 + # - When the file size is smaller than requested Content-range, the Content-range is included in responses with Net::HTTPOK 200 + @chunk_range ||= (chunk_start...(chunk_start + @chunk.bytesize)) + end + + @chunk[chunk_offset..BUFFER_SIZE] + end + + def request + Net::HTTP::Get.new(uri).tap do |request| + request.set_range(chunk_start, BUFFER_SIZE) + end + end + + def chunk_offset + tell % BUFFER_SIZE + end + + def chunk_start + (tell / BUFFER_SIZE) * BUFFER_SIZE + end + + def chunk_end + [chunk_start + BUFFER_SIZE, size].min + end + end +end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3772ef11c7f..7e0398f09af 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module I18n extend self @@ -5,6 +7,7 @@ module Gitlab AVAILABLE_LANGUAGES = { 'en' => 'English', 'es' => 'Español', + 'gl_ES' => 'Galego', 'de' => 'Deutsch', 'fr' => 'Français', 'pt_BR' => 'Português (Brasil)', @@ -21,7 +24,9 @@ module Gitlab 'nl_NL' => 'Nederlands', 'tr_TR' => 'Türkçe', 'id_ID' => 'Bahasa Indonesia', - 'fil_PH' => 'Filipino' + 'fil_PH' => 'Filipino', + 'pl_PL' => 'Polski', + 'cs_CZ' => 'ÄŒeÅ¡tina' }.freeze def available_locales diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 35d57459a3d..3764e379681 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -1,18 +1,29 @@ +# frozen_string_literal: true + module Gitlab module I18n class MetadataEntry attr_reader :entry_data + # Avoid testing too many plurals if `nplurals` was incorrectly set. + # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html + # which mentions special cases for numbers ending in 2 digits + MAX_FORMS_TO_TEST = 101 + def initialize(entry_data) @entry_data = entry_data end - def expected_plurals + def expected_forms return nil unless plural_information plural_information['nplurals'].to_i end + def forms_to_test + @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min + end + private def plural_information diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 7d3ff8c7f58..3e9a035010f 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,6 +1,10 @@ +# frozen_string_literal: true + module Gitlab module I18n class PoLinter + include Gitlab::Utils::StrongMemoize + attr_reader :po_path, :translation_entries, :metadata_entry, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -34,7 +38,7 @@ module Gitlab end @translation_entries = entries.map do |entry_data| - Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms) end nil @@ -48,7 +52,7 @@ module Gitlab translation_entries.each do |entry| errors_for_entry = validate_entry(entry) - errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + errors[entry.msgid] = errors_for_entry if errors_for_entry.any? end errors @@ -62,6 +66,7 @@ module Gitlab validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) validate_unescaped_chars(errors, entry) + validate_translation(errors, entry) errors end @@ -81,35 +86,39 @@ module Gitlab end def validate_number_of_plurals(errors, entry) - return unless metadata_entry&.expected_plurals + return unless metadata_entry&.expected_forms return unless entry.translated? - if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals - errors << "should have #{metadata_entry.expected_plurals} "\ - "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms + errors << "should have #{metadata_entry.expected_forms} "\ + "#{'translations'.pluralize(metadata_entry.expected_forms)}" end end def validate_newlines(errors, entry) - if entry.msgid_contains_newlines? + if entry.msgid_has_multiple_lines? errors << 'is defined over multiple lines, this breaks some tooling.' end - if entry.plural_id_contains_newlines? + if entry.plural_id_has_multiple_lines? errors << 'plural is defined over multiple lines, this breaks some tooling.' end - if entry.translations_contain_newlines? + if entry.translations_have_multiple_lines? errors << 'has translations defined over multiple lines, this breaks some tooling.' end end def validate_variables(errors, entry) if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.msgid) + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) end if entry.has_plural? + validate_variables_in_message(errors, entry.plural_id, entry.plural_id) + entry.plural_translations.each do |translation| validate_variables_in_message(errors, entry.plural_id, translation) end @@ -117,41 +126,98 @@ module Gitlab end def validate_variables_in_message(errors, message_id, message_translation) - message_id = join_message(message_id) required_variables = message_id.scan(VARIABLE_REGEX) validate_unnamed_variables(errors, required_variables) - validate_translation(errors, message_id, required_variables) validate_variable_usage(errors, message_translation, required_variables) end - def validate_translation(errors, message_id, used_variables) + def validate_translation(errors, entry) + Gitlab::I18n.with_locale(locale) do + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) + end + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale}: #{e.message}" + end + + def translate_singular(entry) + used_variables = entry.msgid.scan(VARIABLE_REGEX) variables = fill_in_variables(used_variables) - begin - Gitlab::I18n.with_locale(locale) do - translated = if message_id.include?('|') - FastGettext::Translation.s_(message_id) - else - FastGettext::Translation._(message_id) - end + translation = if entry.msgid.include?('|') + FastGettext::Translation.s_(entry.msgid) + else + FastGettext::Translation._(entry.msgid) + end + + translation % variables if used_variables.any? + end + + def translate_plural(entry) + used_variables = entry.plural_id.scan(VARIABLE_REGEX) + variables = fill_in_variables(used_variables) + + numbers_covering_all_plurals.map do |number| + translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number) + + translation % variables if used_variables.any? + end + end + + def numbers_covering_all_plurals + @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals + end + + def calculate_numbers_covering_all_plurals + required_numbers = [] + discovered_indexes = [] + counter = 0 - translated % variables + while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST + index_for_count = index_for_pluralization(counter) + + unless discovered_indexes.include?(index_for_count) + discovered_indexes << index_for_count + required_numbers << counter end - # `sprintf` could raise an `ArgumentError` when invalid passing something - # other than a Hash when using named variables - # - # `sprintf` could raise `TypeError` when passing a wrong type when using - # unnamed variables - # - # FastGettext::Translation could raise `RuntimeError` (raised as a string), - # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` - # - # `FastGettext::Translation` could raise `ArgumentError` as subclassess - # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` - rescue ArgumentError, TypeError, RuntimeError => e - errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + counter += 1 + end + + required_numbers + end + + def index_for_pluralization(counter) + # This calls the C function that defines the pluralization rule, it can + # return a boolean (`false` represents 0, `true` represents 1) or an integer + # that specifies the plural form to be used for the given number + pluralization_result = Gitlab::I18n.with_locale(locale) do + FastGettext.pluralisation_rule.call(counter) + end + + case pluralization_result + when false + 0 + when true + 1 + else + pluralization_result end end @@ -172,16 +238,20 @@ module Gitlab end def validate_unnamed_variables(errors, variables) - if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } + + if unnamed_variables.any? && named_variables.any? + errors << 'is combining named variables with unnamed variables' + end + + if unnamed_variables.size > 1 errors << 'is combining multiple unnamed variables' end end def validate_variable_usage(errors, translation, required_variables) - translation = join_message(translation) - # We don't need to validate when the message is empty. - # In this case we fall back to the default, which has all the the + # In this case we fall back to the default, which has all the # required variables. return if translation.empty? @@ -205,10 +275,6 @@ module Gitlab def validate_flags(errors, entry) errors << "is marked #{entry.flag}" if entry.flag end - - def join_message(message) - Array(message).join - end end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index e6c95afca7e..19c10b2e402 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module I18n class TranslationEntry @@ -11,11 +13,11 @@ module Gitlab end def msgid - entry_data[:msgid] + @msgid ||= Array(entry_data[:msgid]).join end def plural_id - entry_data[:msgid_plural] + @plural_id ||= Array(entry_data[:msgid_plural]).join end def has_plural? @@ -23,12 +25,11 @@ module Gitlab end def singular_translation - all_translations.first if has_singular_translation? + all_translations.first.to_s if has_singular_translation? end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys) - .reject(&:empty?) + @all_translations ||= translation_entries.map { |translation| Array(translation).join } end def translated? @@ -54,16 +55,16 @@ module Gitlab nplurals > 1 || !has_plural? end - def msgid_contains_newlines? - msgid.is_a?(Array) + def msgid_has_multiple_lines? + entry_data[:msgid].is_a?(Array) end - def plural_id_contains_newlines? - plural_id.is_a?(Array) + def plural_id_has_multiple_lines? + entry_data[:msgid_plural].is_a?(Array) end - def translations_contain_newlines? - all_translations.any? { |translation| translation.is_a?(Array) } + def translations_have_multiple_lines? + translation_entries.any? { |translation| translation.is_a?(Array) } end def msgid_contains_unescaped_chars? @@ -84,6 +85,11 @@ module Gitlab private + def translation_entries + @translation_entries ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + def translation_keys @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3f3f10596c5..d5f94ad04f1 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -1,11 +1,11 @@ +# frozen_string_literal: true + # Detect user based on identifier like -# key-13 or user-36 or last commit +# key-13 or user-36 module Gitlab module Identifier - def identify(identifier, project = nil, newrev = nil) - if identifier.blank? - identify_using_commit(project, newrev) - elsif identifier =~ /\Auser-\d+\Z/ + def identify(identifier) + if identifier =~ /\Auser-\d+\Z/ # git push over http identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ @@ -14,20 +14,8 @@ module Gitlab end end - # Tries to identify a user based on a commit SHA. - def identify_using_commit(project, ref) - return if project.nil? && ref.nil? - - commit = project.commit(ref) - - return if !commit || !commit.author_email - - identify_with_cache(:email, commit.author_email) do - commit.author - end - end - # Tries to identify a user based on a user identifier (e.g. "user-123"). + # rubocop: disable CodeReuse/ActiveRecord def identify_using_user(identifier) user_id = identifier.gsub("user-", "") @@ -35,6 +23,7 @@ module Gitlab User.find_by(id: user_id) end end + # rubocop: enable CodeReuse/ActiveRecord # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). def identify_using_ssh_key(identifier) diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb new file mode 100644 index 00000000000..5b3f30d894a --- /dev/null +++ b/lib/gitlab/import/database_helpers.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module DatabaseHelpers + # Inserts a raw row and returns the ID of the inserted row. + # + # attributes - The attributes/columns to set. + # relation - An ActiveRecord::Relation to use for finding the ID of the row + # when using MySQL. + # rubocop: disable CodeReuse/ActiveRecord + def insert_and_return_id(attributes, relation) + # We use bulk_insert here so we can bypass any queries executed by + # callbacks or validation rules, as doing this wouldn't scale when + # importing very large projects. + result = Gitlab::Database + .bulk_insert(relation.table_name, [attributes], return_ids: true) + + # MySQL doesn't support returning the IDs of a bulk insert in a way that + # is not a pain, so in this case we'll issue an extra query instead. + result.first || + relation.where(iid: attributes[:iid]).limit(1).pluck(:id).first + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb new file mode 100644 index 00000000000..ab3e822a4e9 --- /dev/null +++ b/lib/gitlab/import/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Import + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'importer' + end + end + end +end diff --git a/lib/gitlab/import/merge_request_creator.rb b/lib/gitlab/import/merge_request_creator.rb new file mode 100644 index 00000000000..8291372bba9 --- /dev/null +++ b/lib/gitlab/import/merge_request_creator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# This module is designed for importers that need to create many merge +# requests quickly. When creating merge requests there are a lot of hooks +# that may run, for many different reasons. Many of these hooks (e.g. the ones +# used for rendering Markdown) are completely unnecessary and may even lead to +# transaction timeouts. +# +# To ensure importing merge requests has a minimal impact and can complete in +# a reasonable time we bypass all the hooks by inserting the row and then +# retrieving it. We then only perform the additional work that is strictly +# necessary. +module Gitlab + module Import + class MergeRequestCreator + include ::Gitlab::Import::DatabaseHelpers + include ::Gitlab::Import::MergeRequestHelpers + + attr_accessor :project + + def initialize(project) + @project = project + end + + def execute(attributes) + source_branch_sha = attributes.delete(:source_branch_sha) + target_branch_sha = attributes.delete(:target_branch_sha) + iid = attributes[:iid] + + merge_request, already_exists = create_merge_request_without_hooks(project, attributes, iid) + + if merge_request + insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists) + end + + merge_request + end + end + end +end diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb new file mode 100644 index 00000000000..fa3ff6c3f12 --- /dev/null +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module MergeRequestHelpers + include DatabaseHelpers + + # rubocop: disable CodeReuse/ActiveRecord + def create_merge_request_without_hooks(project, attributes, iid) + # This work must be wrapped in a transaction as otherwise we can leave + # behind incomplete data in the event of an error. This can then lead + # to duplicate key errors when jobs are retried. + MergeRequest.transaction do + # When creating merge requests there are a lot of hooks that may + # run, for many different reasons. Many of these hooks (e.g. the + # ones used for rendering Markdown) are completely unnecessary and + # may even lead to transaction timeouts. + # + # To ensure importing pull requests has a minimal impact and can + # complete in a reasonable time we bypass all the hooks by inserting + # the row and then retrieving it. We then only perform the + # additional work that is strictly necessary. + merge_request_id = insert_and_return_id(attributes, project.merge_requests) + + merge_request = project.merge_requests.reload.find(merge_request_id) + + [merge_request, false] + end + rescue ActiveRecord::InvalidForeignKey + # It's possible the project has been deleted since scheduling this + # job. In this case we'll just skip creating the merge request. + [] + rescue ActiveRecord::RecordNotUnique + # It's possible we previously created the MR, but failed when updating + # the Git data. In this case we'll just continue working on the + # existing row. + [project.merge_requests.find_by(iid: iid), true] + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def insert_or_replace_git_data(merge_request, source_branch_sha, target_branch_sha, already_exists = false) + # These fields are set so we can create the correct merge request + # diffs. + merge_request.source_branch_sha = source_branch_sha + merge_request.target_branch_sha = target_branch_sha + + merge_request.keep_around_commit + + # MR diffs normally use an "after_save" hook to pull data from Git. + # All of this happens in the transaction started by calling + # create/save/etc. This in turn can lead to these transactions being + # held open for much longer than necessary. To work around this we + # first save the diff, then populate it. + diff = + if already_exists + merge_request.merge_request_diffs.take || + merge_request.merge_request_diffs.build + else + merge_request.merge_request_diffs.build + end + + diff.importing = true + diff.save + diff.save_git_content + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..f63a5ece71e 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Gitlab module ImportExport extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index aef371d81eb..d39b6fe5955 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module ImportExport module AfterExportStrategies class BaseAfterExportStrategy + extend Gitlab::ImportExport::CommandLineUtil include ActiveModel::Validations extend Forwardable @@ -24,9 +27,10 @@ module Gitlab end def execute(current_user, project) - return unless project&.export_project_path - @project = project + + return unless @project.export_status == :finished + @current_user = current_user if invalid? @@ -51,9 +55,12 @@ module Gitlab end def self.lock_file_path(project) - return unless project&.export_path + return unless project.export_path || export_file_exists? - File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + lock_path = project.import_export_shared.archive_path + + mkdir_p(lock_path) + File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME) end protected @@ -77,6 +84,10 @@ module Gitlab def log_validation_errors errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end + + def export_file_exists? + project.export_file_exists? + end end end end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb index 4371a7eff56..1b391314a74 100644 --- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport module AfterExportStrategies diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 938664a95a1..b30900f7c61 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport module AfterExportStrategies @@ -23,7 +25,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -38,14 +40,16 @@ module Gitlab private def send_file - export_file = File.open(project.export_project_path) - - Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure export_file.close if export_file end - def send_file_options(export_file) + def export_file + project.export_file.open + end + + def send_file_options { body_stream: export_file, headers: headers @@ -53,7 +57,11 @@ module Gitlab end def headers - { 'Content-Length' => File.size(project.export_project_path).to_s } + { 'Content-Length' => export_size.to_s } + end + + def export_size + project.export_file.file.size end end end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb index 7eabcae2380..37394f46a99 100644 --- a/lib/gitlab/import_export/after_export_strategy_builder.rb +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class AfterExportStrategyBuilder diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 7c9fc5c15bb..93b37b7bc5f 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class AttributeCleaner diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 0c8fda07294..409243e68a5 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class AttributesFinder diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..be1b97bd7a7 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class AvatarRestorer @@ -19,7 +21,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].find { |f| File.file?(f) } end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 998c21e2586..47ca898c690 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,8 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -11,21 +11,15 @@ module Gitlab def save return true unless @project.avatar.exists? - copy_files(avatar_path, avatar_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar' + ).save rescue => e @shared.error(e) false end - - private - - def avatar_export_path - File.join(@shared.export_path, 'avatar', @project.avatar_identifier) - end - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f163db936b..bdecff0931c 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module ImportExport module CommandLineUtil - DEFAULT_MODE = 0700 + UNTAR_MASK = 'u+rwX,go+rX,go-w' + DEFAULT_DIR_MODE = 0700 def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') @@ -12,18 +15,34 @@ module Gitlab end def mkdir_p(path) - FileUtils.mkdir_p(path, mode: DEFAULT_MODE) - FileUtils.chmod(DEFAULT_MODE, path) + FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) + FileUtils.chmod(DEFAULT_DIR_MODE, path) end private + def download_or_copy_upload(uploader, upload_path) + if uploader.upload.local? + copy_files(uploader.path, upload_path) + else + download(uploader.url, upload_path) + end + end + + def download(url, upload_path) + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(url).open, file) + end + end + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end def untar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir})) + execute(%W(chmod -R #{UNTAR_MASK} #{dir})) end def execute(cmd) diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index 788eedf2686..454dc778b6b 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport Error = Class.new(StandardError) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 0f4c3498036..05432f433e7 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -1,23 +1,29 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class FileImporter include Gitlab::ImportExport::CommandLineUtil MAX_RETRIES = 8 + IGNORED_FILENAMES = %w(. ..).freeze def self.import(*args) new(*args).import end - def initialize(archive_file:, shared:) + def initialize(project:, archive_file:, shared:) + @project = project @archive_file = archive_file @shared = shared end def import mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) - remove_symlinks! + remove_symlinks + copy_archive wait_for_archived_file do decompress_archive @@ -26,7 +32,8 @@ module Gitlab @shared.error(e) false ensure - remove_symlinks! + remove_import_file + remove_symlinks end private @@ -50,7 +57,15 @@ module Gitlab result end - def remove_symlinks! + def copy_archive + return if @archive_file + + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + + download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + end + + def remove_symlinks extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end @@ -58,8 +73,12 @@ module Gitlab true end + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} } + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end end end diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb new file mode 100644 index 00000000000..1c62591ed5a --- /dev/null +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `GroupProjectObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class GroupProjectObjectBuilder + def self.build(*args) + Project.transaction do + new(*args).find + end + end + + def initialize(klass, attributes) + @klass = klass < Label ? Label : klass + @attributes = attributes + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + find_object || @klass.create(project_attributes) + end + + private + + def find_object + @klass.where(where_clause).first + end + + def where_clause + @attributes.slice('title').map do |key, value| + scope_clause = table[:project_id].eq(@project.id) + scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + + table[key].eq(value).and(scope_clause) + end.reduce(:or) + end + + def table + @table ||= @klass.arel_table + end + + def project_attributes + @attributes.except('group').tap do |atts| + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + end + end + + def label? + @klass == Label + end + + def milestone? + @klass == Milestone + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = @project.milestones.find_by(iid: @attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end +end diff --git a/lib/gitlab/import_export/hash_util.rb b/lib/gitlab/import_export/hash_util.rb index d4adeeb3797..b6ce89a973b 100644 --- a/lib/gitlab/import_export/hash_util.rb +++ b/lib/gitlab/import_export/hash_util.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class HashUtil diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index da3667faf7a..add7ee58da6 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -19,12 +19,17 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - :issue_assignees - snippets: - :award_emoji - notes: :author - - :releases + - releases: + - :author + - :links - project_members: - :user - merge_requests: @@ -45,7 +50,10 @@ project_tree: - milestone: - events: - :push_event_payload - - pipelines: + - resource_label_events: + - label: + :priorities + - ci_pipelines: - notes: - :author - events: @@ -56,7 +64,6 @@ project_tree: - :triggers - :pipeline_schedules - :services - - :hooks - protected_branches: - :merge_access_levels - :push_access_levels @@ -64,8 +71,10 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :prometheus_metrics - :project_badges - :ci_cd_settings + - :error_tracking_setting # Only include the following attributes for the models specified. included_attributes: @@ -85,18 +94,18 @@ excluded_attributes: - :path - :namespace_id - :creator_id + - :pool_repository_id - :import_url - :import_status - :avatar - :import_type - :import_source - - :import_error - :mirror - :runners_token + - :runners_token_encrypted - :repository_storage - :repository_read_only - :lfs_enabled - - :import_jid - :created_at - :updated_at - :id @@ -107,6 +116,17 @@ excluded_attributes: - :storage_version - :remote_mirror_available_overridden - :description_html + - :repository_languages + - :bfg_object_map + namespaces: + - :runners_token + - :runners_token_encrypted + project_import_state: + - :last_error + - :jid + prometheus_metrics: + - :common + - :identifier snippets: - :expired_at merge_request_diff: @@ -125,13 +145,28 @@ excluded_attributes: statuses: - :trace - :token + - :token_encrypted - :when - :artifacts_file - :artifacts_metadata + - :commands push_event_payload: - :event_id project_badges: - :group_id + resource_label_events: + - :reference + - :reference_html + - :epic_id + runners: + - :token + - :token_encrypted + services: + - :template + error_tracking_setting: + - :encrypted_token + - :encrypted_token_iv + - :enabled methods: labels: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 63cab07324a..767f1b5de0e 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class Importer @@ -35,7 +37,8 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + Gitlab::ImportExport::FileImporter.import(project: @project, + archive_file: @archive_file, shared: @shared) end @@ -91,7 +94,12 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + upload = @project.import_export_upload + + return unless upload&.import_file&.file + + upload.remove_import_file! + upload.save! end def overwrite_project diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index b48f63bcd7e..477499e1688 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport # Generates a hash that conforms with http://apidock.com/rails/Hash/to_json diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index b28c3c161b7..345c7880e30 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class LfsRestorer diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 29410e2331c..954f6f00078 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class LfsSaver diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 8b8e48aac76..6be95a16513 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class MembersMapper @@ -45,9 +47,9 @@ module Gitlab end def ensure_default_member! - @project.project_members.destroy_all + @project.project_members.destroy_all # rubocop: disable DestroyAll - ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) end def add_team_member(member, existing_user = nil) diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index f3d7407383c..040a70d6775 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class MergeRequestParser @@ -25,8 +27,13 @@ module Gitlab @project.repository.create_branch(@merge_request.target_branch, @merge_request.target_branch_sha) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1295 def fetch_ref - @project.repository.fetch_ref(@project.repository, source_ref: @diff_head_sha, target_ref: @merge_request.source_branch) + target_ref = Gitlab::Git::BRANCH_REF_PREFIX + @merge_request.source_branch + + unless @project.repository.fetch_source_branch!(@project.repository, @diff_head_sha, target_ref) + Rails.logger.warn("Import/Export warning: Failed to create #{target_ref} for MR: #{@merge_request.iid}") + end end def branch_exists?(branch_name) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4eb67fbe11e..51001750a6c 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @@ -24,6 +26,8 @@ module Gitlab @project_members = @tree_hash.delete('project_members') + RelationRenameService.rename(@tree_hash) + ActiveRecord::Base.uncached do ActiveRecord::Base.no_touching do create_relations @@ -70,12 +74,23 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + remove_group_models(relation_hash) if relation_hash.is_a?(Array) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Remove project models that became group models as we found them at group level. + # This no longer required saving them at the root project level. + # For example, in the case of an existing group label that matched the title. + def remove_group_models(relation_hash) + relation_hash.reject! do |value| + GROUP_MODELS.include?(value.class) && value.group_id + end + end + def default_relation_list reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] @@ -83,13 +98,16 @@ module Gitlab end def restore_project - @project.update_columns(project_params) + Gitlab::Timeless.timeless(@project) do + @project.update(project_params) + end + @project end def project_params @project_params ||= begin - attrs = json_params.merge(override_params) + attrs = json_params.merge(override_params).merge(visibility_level) # Cleaning all imported and overridden params Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs, @@ -109,6 +127,13 @@ module Gitlab end end + def visibility_level + level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level + level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level + + { 'visibility_level' => level } + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ @@ -122,16 +147,25 @@ module Gitlab return if tree_hash[relation_key].blank? tree_array = [tree_hash[relation_key]].flatten + null_iid_pipelines = [] # Avoid keeping a possible heavy object in memory once we are done with it - while relation_item = tree_array.shift + while relation_item = (tree_array.shift || null_iid_pipelines.shift) + if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any? + # Move pipelines with NULL IIDs to the end + # so they don't clash with existing IIDs. + null_iid_pipelines << relation_item + + next + end + # The transaction at this level is less speedy than one single transaction # But we can't have it in the upper level or GC won't get rid of the AR objects # after we save the batch. Project.transaction do process_sub_relation(relation, relation_item) - # For every subrelation that hangs from Project, save the associated records alltogether + # For every subrelation that hangs from Project, save the associated records altogether # This effectively batches all records per subrelation item, only keeping those in memory # We have to keep in mind that more batch granularity << Memory, but >> Slowness if save @@ -170,7 +204,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), + relation_hash: relation_hash, members_mapper: members_mapper, user: @user, project: @restored_project, @@ -180,24 +214,16 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash, relation_type) - if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) - params = {} - params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] - params['project_id'] = restored_project.id if relation_hash['project_id'] - else - params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } - end - - relation_hash.merge(params) - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) + end + + def nil_iid_pipeline?(relation_key, relation_item) + relation_key == 'ci_pipelines' && relation_item['iid'].nil? end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 5510c0b8b2f..2255635acdf 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class ProjectTreeSaver @@ -32,6 +34,8 @@ module Gitlab project_json['project_members'] += group_members_json + RelationRenameService.add_new_associations(project_json) + project_json.to_json end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index e621c40fc7a..bc0d18e03fa 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class Reader diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index c5cf290f191..099b488f68e 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,13 +1,17 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class RelationFactory OVERRIDES = { snippets: :project_snippets, + ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + runners: 'Ci::Runner', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -19,7 +23,9 @@ module Gitlab custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge', metrics: 'MergeRequest::Metrics', - ci_cd_settings: 'ProjectCiCdSetting' }.freeze + ci_cd_settings: 'ProjectCiCdSetting', + error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', + links: 'Releases::Link' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze @@ -31,7 +37,7 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze - TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze def self.create(*args) new(*args).create @@ -47,13 +53,15 @@ module Gitlab end def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) - @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_name = self.class.overrides[relation_sym] || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @user = user @project = project @imported_object_retries = 0 + @relation_hash['project_id'] = @project.id + # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, @@ -74,23 +82,25 @@ module Gitlab generate_imported_object end + def self.overrides + OVERRIDES + end + private def setup_models case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when :project_label, :project_labels then setup_label - when :milestone, :milestones then setup_milestone - when 'Ci::Pipeline' then setup_pipeline - else - @relation_hash['project_id'] = @project.id end update_user_references update_project_references + update_group_references remove_duplicate_assignees + setup_pipeline if @relation_name == 'Ci::Pipeline' + reset_tokens! remove_encrypted_attributes! end @@ -141,6 +151,7 @@ module Gitlab if BUILD_MODELS.include?(@relation_name) @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') + @relation_hash.delete('commands') imported_object elsif @relation_name == :merge_requests @@ -151,39 +162,23 @@ module Gitlab end def update_project_references - project_id = @relation_hash.delete('project_id') - # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID end - # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id - @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] end def same_source_and_target? @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def setup_label - # If there's no group, move the label to a project label - if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] - @relation_hash['project_id'] = nil - @relation_name = :group_label - else - @relation_hash['group_id'] = nil - @relation_hash['type'] = 'ProjectLabel' - end - end + def update_group_references + return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless @relation_hash['group_id'] - def setup_milestone - if @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group.id - else - @relation_hash['project_id'] = @project.id - end + @relation_hash['group_id'] = @project.group&.id end def reset_tokens! @@ -223,7 +218,7 @@ module Gitlab def update_note_for_missing_author(author_name) @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" end def admin_user? @@ -271,15 +266,7 @@ module Gitlab end def existing_object - @existing_object ||= - begin - existing_object = find_or_create_object! - - # Done in two steps, as MySQL behaves differently than PostgreSQL using - # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update!(parsed_relation_hash) - existing_object - end + @existing_object ||= find_or_create_object! end def unknown_service? @@ -288,29 +275,16 @@ module Gitlab end def find_or_create_object! - finder_attributes = if @relation_name == :group_label - %w[title group_id] - elsif parsed_relation_hash['project_id'] - %w[title project_id] - else - %w[title group_id] - end - - finder_hash = parsed_relation_hash.slice(*finder_attributes) - - if label? - label = relation_class.find_or_initialize_by(finder_hash) - parsed_relation_hash.delete('priorities') if label.persisted? - - label.save! - label - else - relation_class.find_or_create_by(finder_hash) + return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + hash['group'] = @project.group if relation_class.attribute_method?('group_id') + hash['project'] = @project + hash.delete('project_id') end - end - def label? - @relation_name.to_s.include?('label') + GroupProjectObjectBuilder.build(relation_class, finder_hash) end end end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb new file mode 100644 index 00000000000..179bde5e21e --- /dev/null +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# This class is intended to help with relation renames within Gitlab versions +# and allow compatibility between versions. +# If you have to change one relationship name that is imported/exported, +# you should add it to the RENAMES constant indicating the old name and the +# new one. +# The behavior of these renamed relationships should be transient and it should +# only last one release until you completely remove the renaming from the list. +# +# When importing, this class will check the project hash and: +# - if only the old relationship name is found, it will rename it with the new one +# - if only the new relationship name is found, it will do nothing +# - if it finds both, it will use the new relationship data +# +# When exporting, this class will duplicate the keys in the resulting file. +# This way, if we open the file in an old version of the exporter it will work +# and also it will with the newer versions. +module Gitlab + module ImportExport + class RelationRenameService + RENAMES = { + 'pipelines' => 'ci_pipelines' # Added in 11.6, remove in 11.7 + }.freeze + + def self.rename(tree_hash) + return unless tree_hash&.present? + + RENAMES.each do |old_name, new_name| + old_entry = tree_hash.delete(old_name) + + next if tree_hash[new_name] + next unless old_entry + + tree_hash[new_name] = old_entry + end + end + + def self.add_new_associations(tree_hash) + RENAMES.each do |old_name, new_name| + next if tree_hash.key?(old_name) + + tree_hash[old_name] = tree_hash[new_name] + end + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 5a9bbceac67..91167a9c4fb 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -1,8 +1,9 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - include Gitlab::ShellAdapter def initialize(project:, shared:, path_to_bundle:) @project = project diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 695462c7dd2..a60618dfcec 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class RepoSaver @@ -26,10 +28,6 @@ module Gitlab @shared.error(e) false end - - def path_to_repo - @project.repository.path_to_repo - end end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 2daeba90a51..72f575db095 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class Saver @@ -15,15 +17,20 @@ module Gitlab def save if compress_and_save remove_export_path + Rails.logger.info("Saved project export #{archive_file}") - archive_file + + save_upload else - @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}")) + @shared.error(Gitlab::ImportExport::Error.new(error_message)) false end rescue => e @shared.error(e) false + ensure + remove_archive + remove_export_path end private @@ -36,9 +43,25 @@ module Gitlab FileUtils.rm_rf(@shared.export_path) end + def remove_archive + FileUtils.rm_rf(@shared.archive_path) + end + def archive_file @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end + + def save_upload + upload = ImportExportUpload.find_or_initialize_by(project: @project) + + File.open(archive_file) { |file| upload.export_file = file } + + upload.save! + end + + def error_message + "Unable to save #{archive_file} into #{@shared.export_path}." + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 6d7c36ce38b..947caaaefee 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class Shared @@ -6,6 +8,7 @@ module Gitlab def initialize(project) @project = project @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count @@ -21,19 +24,16 @@ module Gitlab end def error(error) - error_out(error.message, caller[0].dup) - add_error_message(error.message) + log_error(message: error.message, caller: caller[0].dup) + log_debug(backtrace: error.backtrace&.join("\n")) + + Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data) - # Debug: - if error.backtrace - Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}") - else - Rails.logger.error("No backtrace found") - end + add_error_message(error.message) end - def add_error_message(error_message) - @errors << error_message + def add_error_message(message) + @errors << filtered_error_message(message) end def after_export_in_progress? @@ -50,8 +50,25 @@ module Gitlab @project.disk_path end - def error_out(message, caller) - Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + def log_error(details) + @logger.error(log_base_data.merge(details)) + end + + def log_debug(details) + @logger.debug(log_base_data.merge(details)) + end + + def log_base_data + { + importer: 'Import/Export', + import_jid: @project&.import_state&.import_jid, + project_id: @project&.id, + project_path: @project&.full_path + } + end + + def filtered_error_message(message) + Projects::ImportErrorFilter.filter_message(message) end def after_export_lock_file diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb index bcdd9c12c85..3fafb01c37c 100644 --- a/lib/gitlab/import_export/statistics_restorer.rb +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class StatisticsRestorer diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb new file mode 100644 index 00000000000..e232198150a --- /dev/null +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class UploadsManager + include Gitlab::ImportExport::CommandLineUtil + + UPLOADS_BATCH_SIZE = 100 + + def initialize(project:, shared:, relative_export_path: 'uploads') + @project = project + @shared = shared + @relative_export_path = relative_export_path + end + + def save + copy_project_uploads + + true + rescue => e + @shared.error(e) + false + end + + def restore + Dir["#{uploads_export_path}/**/*"].each do |upload| + next if File.directory?(upload) + + add_upload(upload) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def add_upload(upload) + uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys + + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h + end + + def copy_project_uploads + each_uploader do |uploader| + next unless uploader.file + + if uploader.upload.local? + next unless uploader.upload.exist? + + copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) + else + download_and_copy(uploader) + end + end + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) + end + + def each_uploader + avatar_path = @project.avatar&.upload&.path + + if @relative_export_path == 'avatar' + yield(@project.avatar) + else + project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload| + yield(upload.build_uploader) + end + end + end + + def project_uploads_except_avatar(avatar_path) + return @project.uploads unless avatar_path + + @project.uploads.where("path != ?", avatar_path) + end + + def download_and_copy(upload) + secret = upload.try(:secret) || '' + upload_path = File.join(uploads_export_path, secret, upload.filename) + + mkdir_p(File.join(uploads_export_path, secret)) + + download_or_copy_upload(upload, upload_path) + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index df19354b76e..5f422dcbefa 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -1,10 +1,13 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - return true unless File.directory?(uploads_export_path) - - copy_files(uploads_export_path, uploads_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 2f08dda55fd..be1066c30b2 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,29 +1,22 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared end def save - return true unless File.directory?(uploads_path) - - copy_files(uploads_path, uploads_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index bd3c3ee3b2f..6d978d00ea5 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class VersionChecker diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 7cf88298642..8230c0f1e77 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class VersionSaver diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 5fa2e101e29..7303bcf61a4 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class WikiRepoSaver < RepoSaver @@ -22,12 +24,8 @@ module Gitlab "project.wiki.bundle" end - def path_to_repo - @wiki.repository.path_to_repo - end - def wiki_repository_exists? - File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + @wiki.repository.exists? && !@wiki.repository.empty? end end end diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb index f33bfb332ab..28b5e7449cd 100644 --- a/lib/gitlab/import_export/wiki_restorer.rb +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ImportExport class WikiRestorer < RepoRestorer diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index 4e611e7f16c..d4ba4d1181d 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ImportFormatter def comment(author, date, body) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 60d5fa4d29a..67952ca0f2d 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitlab::ImportSources module # # Define import sources that can be used @@ -8,37 +10,43 @@ module Gitlab ImportSource = Struct.new(:name, :title, :importer) # We exclude `bare_repository` here as it has no import class associated - ImportTable = [ - ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), - ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), - ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), - ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), - ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), - ImportSource.new('git', 'Repo by URL', nil), - ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer) + IMPORT_TABLE = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), + ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), + ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), + ImportSource.new('manifest', 'Manifest file', nil) ].freeze class << self def options - @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + Hash[import_table.map { |importer| [importer.title, importer.name] }] end def values - @values ||= ImportTable.map(&:name) + import_table.map(&:name) end def importer_names - @importer_names ||= ImportTable.select(&:importer).map(&:name) + import_table.select(&:importer).map(&:name) end def importer(name) - ImportTable.find { |import_source| import_source.name == name }.importer + import_table.find { |import_source| import_source.name == name }.importer end def title(name) options.key(name) end + + def import_table + IMPORT_TABLE + end end end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d323cb9dadf..cc0c633b943 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze - WILDCARD_PLACEHOLDER = '%{key}'.freeze + UNSUBSCRIBE_SUFFIX = '-unsubscribe'.freeze + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe'.freeze + WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self def enabled? @@ -20,6 +23,7 @@ module Gitlab config.address.sub(WILDCARD_PLACEHOLDER, key) end + # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com def unsubscribe_address(key) config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index f85b6e9197f..e4f0e9d2c73 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # # Calculates the fingerprint of a given key without using diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 0c9de72329c..351d15605e0 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module IssuableMetadata def issuable_meta_data(issuable_collection, collection_type) diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb index d392214867a..42bbfb32d0b 100644 --- a/lib/gitlab/issuable_sorter.rb +++ b/lib/gitlab/issuable_sorter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module IssuableSorter class << self diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 505810964bc..659fb1472d2 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -1,7 +1,9 @@ +# frozen_string_literal: true + module Gitlab # Class for counting and caching the number of issuables per state. class IssuablesCountForState - # The name of the RequestStore cache key. + # The name of the Gitlab::SafeRequestStore cache key. CACHE_KEY = :issuables_count_for_state # The state values that can be safely casted to a Symbol. @@ -10,12 +12,7 @@ module Gitlab # finder - The finder class to use for retrieving the issuables. def initialize(finder) @finder = finder - @cache = - if RequestStore.active? - RequestStore[CACHE_KEY] ||= initialize_cache - else - initialize_cache - end + @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end def for_state_or_opened(state = nil) diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index b8ca7f2f55f..17c9cb969df 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class IssuesLabels class << self diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index f7a8eae0be4..e97e961771c 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. # diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb new file mode 100644 index 00000000000..1adf83739ad --- /dev/null +++ b/lib/gitlab/json_cache.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + class JsonCache + attr_reader :backend, :cache_key_with_version, :namespace + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_with_version = options.fetch(:cache_key_with_version, true) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def cache_key(key) + expanded_cache_key = [namespace, key].compact + + if cache_key_with_version + expanded_cache_key << Rails.version + end + + expanded_cache_key.join(':') + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = backend.read(cache_key(key)) + value = parse_value(value, klass) if value + value + end + + def write(key, value, options = nil) + backend.write(cache_key(key), value.to_json, options) + end + + def fetch(key, options = {}, &block) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + def parse_value(raw, klass) + value = ActiveSupport::JSON.decode(raw) + + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + rescue ActiveSupport::JSON.parse_error + nil + end + + def parse_entry(raw, klass) + klass.new(raw) if valid_entry?(raw, klass) + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.map { |value| parse_entry(value, klass) }.compact + end + end +end diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb new file mode 100644 index 00000000000..a5a5759cc89 --- /dev/null +++ b/lib/gitlab/json_logger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + class JsonLogger < ::Gitlab::Logger + def self.file_name_noext + raise NotImplementedError + end + + def format_message(severity, timestamp, progname, message) + data = {} + data[:severity] = severity + data[:time] = timestamp.utc.iso8601(3) + data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id + + case message + when String + data[:message] = message + when Hash + data.merge!(message) + end + + data.to_json + "\n" + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index da43bd0af4b..a9957a85d48 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -1,6 +1,12 @@ +# frozen_string_literal: true + module Gitlab # Helper methods to do with Kubernetes network services & resources module Kubernetes + def self.build_header_hash + Hash.new { |h, k| h[k] = [] } + end + # This is the comand that is run to start a terminal session. Kubernetes # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` EXEC_COMMAND = URI.encode_www_form( @@ -37,13 +43,14 @@ module Gitlab selectors: { pod: pod_name, container: container["name"] }, url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], - headers: Hash.new { |h, k| h[k] = [] }, + headers: ::Gitlab::Kubernetes.build_header_hash, created_at: created_at } end end def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) + terminal[:headers] ||= ::Gitlab::Kubernetes.build_header_hash terminal[:headers]['Authorization'] << "Bearer #{token}" terminal[:max_session_time] = max_session_time terminal[:ca_pem] = ca_pem if ca_pem.present? @@ -78,6 +85,8 @@ module Gitlab end def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) + return unless token.present? + config = { apiVersion: 'v1', clusters: [ @@ -106,7 +115,7 @@ module Gitlab kubeconfig_embed_ca_pem(config, ca_pem) if ca_pem - config.deep_stringify_keys + YAML.dump(config.deep_stringify_keys) end private diff --git a/lib/gitlab/kubernetes/cluster_role_binding.rb b/lib/gitlab/kubernetes/cluster_role_binding.rb new file mode 100644 index 00000000000..ebea8aff5be --- /dev/null +++ b/lib/gitlab/kubernetes/cluster_role_binding.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ClusterRoleBinding + attr_reader :name, :cluster_role_name, :subjects + + def initialize(name, cluster_role_name, subjects) + @name = name + @cluster_role_name = cluster_role_name + @subjects = subjects + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.roleRef = role_ref + resource.subjects = subjects + end + end + + private + + def metadata + { name: name } + end + + def role_ref + { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: cluster_role_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/config_map.rb b/lib/gitlab/kubernetes/config_map.rb index 95e1054919d..0bcaaa03974 100644 --- a/lib/gitlab/kubernetes/config_map.rb +++ b/lib/gitlab/kubernetes/config_map.rb @@ -1,21 +1,27 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes class ConfigMap - def initialize(name, values) + def initialize(name, files) @name = name - @values = values + @files = files end def generate resource = ::Kubeclient::Resource.new resource.metadata = metadata - resource.data = { values: values } + resource.data = files resource end + def config_map_name + "values-content-configuration-#{name}" + end + private - attr_reader :name, :values + attr_reader :name, :files def metadata { @@ -25,10 +31,6 @@ module Gitlab } end - def config_map_name - "values-content-configuration-#{name}" - end - def namespace Gitlab::Kubernetes::Helm::NAMESPACE end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 0f0588b8b23..bbac15c7710 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,8 +1,14 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.7.0'.freeze + HELM_VERSION = '2.12.2'.freeze + KUBECTL_VERSION = '1.11.0'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze + SERVICE_ACCOUNT = 'tiller'.freeze + CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze + CLUSTER_ROLE = 'cluster-admin'.freeze end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 2edd34109ba..b9903e37f40 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm @@ -8,9 +10,23 @@ module Gitlab end def install(command) - @namespace.ensure_exists! - create_config_map(command) if command.config_map? - @kubeclient.create_pod(command.pod_resource) + namespace.ensure_exists! + + create_service_account(command) + create_cluster_role_binding(command) + create_config_map(command) + + delete_pod!(command.pod_name) + kubeclient.create_pod(command.pod_resource) + end + + def update(command) + namespace.ensure_exists! + + update_config_map(command) + + delete_pod!(command.pod_name) + kubeclient.create_pod(command.pod_resource) end ## @@ -20,25 +36,87 @@ module Gitlab # # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" # - def installation_status(pod_name) - @kubeclient.get_pod(pod_name, @namespace.name).status.phase + def status(pod_name) + kubeclient.get_pod(pod_name, namespace.name).status.phase end - def installation_log(pod_name) - @kubeclient.get_pod_log(pod_name, @namespace.name).body + def log(pod_name) + kubeclient.get_pod_log(pod_name, namespace.name).body end - def delete_installation_pod!(pod_name) - @kubeclient.delete_pod(pod_name, @namespace.name) + def delete_pod!(pod_name) + kubeclient.delete_pod(pod_name, namespace.name) + rescue ::Kubeclient::ResourceNotFoundError + # no-op + end + + def get_config_map(config_map_name) + namespace.ensure_exists! + + kubeclient.get_config_map(config_map_name, namespace.name) end private + attr_reader :kubeclient, :namespace + def create_config_map(command) command.config_map_resource.tap do |config_map_resource| - @kubeclient.create_config_map(config_map_resource) + if config_map_exists?(config_map_resource) + kubeclient.update_config_map(config_map_resource) + else + kubeclient.create_config_map(config_map_resource) + end + end + end + + def update_config_map(command) + command.config_map_resource.tap do |config_map_resource| + kubeclient.update_config_map(config_map_resource) end end + + def create_service_account(command) + command.service_account_resource.tap do |service_account_resource| + break unless service_account_resource + + if service_account_exists?(service_account_resource) + kubeclient.update_service_account(service_account_resource) + else + kubeclient.create_service_account(service_account_resource) + end + end + end + + def create_cluster_role_binding(command) + command.cluster_role_binding_resource.tap do |cluster_role_binding_resource| + break unless cluster_role_binding_resource + + if cluster_role_binding_exists?(cluster_role_binding_resource) + kubeclient.update_cluster_role_binding(cluster_role_binding_resource) + else + kubeclient.create_cluster_role_binding(cluster_role_binding_resource) + end + end + end + + def config_map_exists?(resource) + kubeclient.get_config_map(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def service_account_exists?(resource) + kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def cluster_role_binding_exists?(resource) + kubeclient.get_cluster_role_binding(resource.metadata.name) + rescue ::Kubeclient::ResourceNotFoundError + false + end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 3d778da90c7..2bcb428b25d 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -1,42 +1,66 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm - class BaseCommand - attr_reader :name - - def initialize(name) - @name = name - end - + module BaseCommand def pod_resource - Gitlab::Kubernetes::Helm::Pod.new(self, namespace).generate + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate end def generate_script <<~HEREDOC - set -eo pipefail - ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2) - echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - apk add -U ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ + set -xeo pipefail HEREDOC end - def config_map? - false - end - def pod_name "install-#{name}" end + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, files).generate + end + + def service_account_resource + nil + end + + def cluster_role_binding_resource + nil + end + + def file_names + files.keys + end + + def name + raise "Not implemented" + end + + def rbac? + raise "Not implemented" + end + + def files + raise "Not implemented" + end + private + def files_dir + "/data/helm/#{name}/config" + end + def namespace Gitlab::Kubernetes::Helm::NAMESPACE end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end end end end diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb new file mode 100644 index 00000000000..598714e0874 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/certificate.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true +module Gitlab + module Kubernetes + module Helm + class Certificate + INFINITE_EXPIRY = 1000.years + SHORT_EXPIRY = 30.minutes + + attr_reader :key, :cert + + def key_string + @key.to_s + end + + def cert_string + @cert.to_pem + end + + def self.from_strings(key_string, cert_string) + key = OpenSSL::PKey::RSA.new(key_string) + cert = OpenSSL::X509::Certificate.new(cert_string) + new(key, cert) + end + + def self.generate_root + _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) + end + + def issue(expires_in: SHORT_EXPIRY) + self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) + end + + private + + def self._issue(signed_by:, expires_in:, certificate_authority:) + key = OpenSSL::PKey::RSA.new(4096) + public_key = key.public_key + + subject = OpenSSL::X509::Name.parse("/C=US") + + cert = OpenSSL::X509::Certificate.new + cert.subject = subject + + cert.issuer = signed_by&.cert&.subject || subject + + cert.not_before = Time.now + cert.not_after = expires_in.from_now + cert.public_key = public_key + cert.serial = 0x0 + cert.version = 2 + + if certificate_authority + extension_factory = OpenSSL::X509::ExtensionFactory.new + extension_factory.subject_certificate = cert + extension_factory.issuer_certificate = cert + cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) + cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) + cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) + end + + cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + + new(key, cert) + end + + def initialize(key, cert) + @key = key + @cert = cert + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb new file mode 100644 index 00000000000..9940272a8bf --- /dev/null +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module ClientCommand + def init_command + # Here we are always upgrading to the latest version of Tiller when + # installing an app. We ensure the helm version stored in the + # database is correct by also updating this after transition to + # :installed,:updated in Clusters::Concerns::ApplicationStatus + 'helm init --upgrade' + end + + def wait_for_tiller_command + # This is necessary to give Tiller time to restart after upgrade. + # Ideally we'd be able to use --wait but cannot because of + # https://github.com/helm/helm/issues/4855 + 'for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done' + end + + def repository_command + ['helm', 'repo', 'add', name, repository].shelljoin if repository + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index a02e64561f6..88ed8572ffc 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -1,17 +1,81 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm - class InitCommand < BaseCommand + class InitCommand + include BaseCommand + + attr_reader :name, :files + + def initialize(name:, files:, rbac:) + @name = name + @files = files + @rbac = rbac + end + def generate_script super + [ init_helm_command ].join("\n") end + def rbac? + @rbac + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + private def init_helm_command - "helm init >/dev/null" + command = %w[helm init] + init_command_flags + + command.shelljoin + end + + def init_command_flags + tls_flags + optional_service_account_flag + end + + def tls_flags + [ + '--tiller-tls', + '--tiller-tls-verify', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tiller-tls-cert', "#{files_dir}/cert.pem", + '--tiller-tls-key', "#{files_dir}/key.pem" + ] + end + + def optional_service_account_flag + return [] unless rbac? + + ['--service-account', service_account_name] + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 30af3e97b4a..a1ab5e048ac 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -1,46 +1,97 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm - class InstallCommand < BaseCommand - attr_reader :name, :chart, :repository, :values + class InstallCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall - def initialize(name, chart:, values:, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) @name = name @chart = chart - @values = values + @version = version + @rbac = rbac + @files = files @repository = repository + @preinstall = preinstall + @postinstall = postinstall end def generate_script super + [ init_command, + wait_for_tiller_command, repository_command, - script_command + repository_update_command, + preinstall_command, + install_command, + postinstall_command ].compact.join("\n") end - def config_map? - true + def rbac? + @rbac + end + + private + + def repository_update_command + 'helm repo update' if repository end - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, values).generate + def install_command + command = ['helm', 'install', chart] + install_command_flags + + command.shelljoin end - private + def preinstall_command + preinstall.join("\n") if preinstall + end + + def postinstall_command + postinstall.join("\n") if postinstall + end + + def install_command_flags + name_flag = ['--name', name] + namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + value_flag = ['-f', "/data/helm/#{name}/config/values.yaml"] - def init_command - 'helm init --client-only >/dev/null' + name_flag + + optional_tls_flags + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag end - def repository_command - "helm repo add #{name} #{repository}" if repository + def rbac_create_flag + if rbac? + %w[--set rbac.create=true,rbac.enabled=true] + else + %w[--set rbac.create=false,rbac.enabled=false] + end end - def script_command - <<~HEREDOC - helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null - HEREDOC + def optional_version_flag + return [] unless version + + ['--version', version] + end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] end end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 1e12299eefd..75484f80070 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -1,31 +1,33 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Helm class Pod - def initialize(command, namespace_name) + def initialize(command, namespace_name, service_account_name: nil) @command = command @namespace_name = namespace_name + @service_account_name = service_account_name end def generate spec = { containers: [container_specification], restartPolicy: 'Never' } - if command.config_map? - spec[:volumes] = volumes_specification - spec[:containers][0][:volumeMounts] = volume_mounts_specification - end + spec[:volumes] = volumes_specification + spec[:containers][0][:volumeMounts] = volume_mounts_specification + spec[:serviceAccountName] = service_account_name if service_account_name ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end private - attr_reader :command, :namespace_name, :kubeclient, :config_map + attr_reader :command, :namespace_name, :service_account_name def container_specification { name: 'helm', - image: 'alpine:3.6', + image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}", env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) @@ -61,7 +63,7 @@ module Gitlab name: 'configuration-volume', configMap: { name: "values-content-configuration-#{command.name}", - items: [{ key: 'values', path: 'values.yaml' }] + items: command.file_names.map { |name| { key: name, path: name } } } } ] diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb new file mode 100644 index 00000000000..9daffc138b5 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class UpgradeCommand + include BaseCommand + include ClientCommand + + attr_reader :name, :chart, :version, :repository, :files + + def initialize(name, chart:, files:, rbac:, version: nil, repository: nil) + @name = name + @chart = chart + @rbac = rbac + @version = version + @files = files + @repository = repository + end + + def generate_script + super + [ + init_command, + wait_for_tiller_command, + repository_command, + script_command + ].compact.join("\n") + end + + def rbac? + @rbac + end + + def pod_name + "upgrade-#{name}" + end + + private + + def script_command + upgrade_flags = "#{optional_version_flag}#{optional_tls_flags}" \ + " --reset-values" \ + " --install" \ + " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ + " -f /data/helm/#{name}/config/values.yaml" + + "helm upgrade #{name} #{chart}#{upgrade_flags}" + end + + def optional_version_flag + " --version #{version}" if version + end + + def optional_tls_flags + return unless files.key?(:'ca.pem') + + " --tls" \ + " --tls-ca-cert #{files_dir}/ca.pem" \ + " --tls-cert #{files_dir}/cert.pem" \ + " --tls-key #{files_dir}/key.pem" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb new file mode 100644 index 00000000000..624c2c67551 --- /dev/null +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require 'uri' + +module Gitlab + module Kubernetes + # Wrapper around Kubeclient::Client to dispatch + # the right message to the client that can respond to the message. + # We must have a kubeclient for each ApiGroup as there is no + # other way to use the Kubeclient gem. + # + # See https://github.com/abonas/kubeclient/issues/348. + class KubeClient + include Gitlab::Utils::StrongMemoize + + SUPPORTED_API_GROUPS = { + core: { group: 'api', version: 'v1' }, + rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' }, + extensions: { group: 'apis/extensions', version: 'v1beta1' }, + knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' } + }.freeze + + SUPPORTED_API_GROUPS.each do |name, params| + client_method_name = "#{name}_client".to_sym + + define_method(client_method_name) do + strong_memoize(client_method_name) do + build_kubeclient(params[:group], params[:version]) + end + end + end + + # Core API methods delegates to the core api group client + delegate :get_pods, + :get_secrets, + :get_config_map, + :get_namespace, + :get_pod, + :get_secret, + :get_service, + :get_service_account, + :delete_pod, + :create_config_map, + :create_namespace, + :create_pod, + :create_secret, + :create_service_account, + :update_config_map, + :update_secret, + :update_service_account, + to: :core_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client + delegate :create_cluster_role_binding, + :get_cluster_role_binding, + :update_cluster_role_binding, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client + delegate :create_role_binding, + :get_role_binding, + :update_role_binding, + to: :rbac_client + + # Deployments resource is currently on the apis/extensions api group + delegate :get_deployments, + to: :extensions_client + + # non-entity methods that can only work with the core client + # as it uses the pods/log resource + delegate :get_pod_log, + :watch_pod_log, + to: :core_client + + attr_reader :api_prefix, :kubeclient_options + + # We disable redirects through 'http_max_redirects: 0', + # so that KubeClient does not follow redirects and + # expose internal services. + def initialize(api_prefix, **kubeclient_options) + @api_prefix = api_prefix + @kubeclient_options = kubeclient_options.merge(http_max_redirects: 0) + end + + def create_or_update_cluster_role_binding(resource) + if cluster_role_binding_exists?(resource) + update_cluster_role_binding(resource) + else + create_cluster_role_binding(resource) + end + end + + def create_or_update_role_binding(resource) + if role_binding_exists?(resource) + update_role_binding(resource) + else + create_role_binding(resource) + end + end + + def create_or_update_service_account(resource) + if service_account_exists?(resource) + update_service_account(resource) + else + create_service_account(resource) + end + end + + def create_or_update_secret(resource) + if secret_exists?(resource) + update_secret(resource) + else + create_secret(resource) + end + end + + private + + def cluster_role_binding_exists?(resource) + get_cluster_role_binding(resource.metadata.name) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def role_binding_exists?(resource) + get_role_binding(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def service_account_exists?(resource) + get_service_account(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def secret_exists?(resource) + get_secret(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false + end + + def build_kubeclient(api_group, api_version) + ::Kubeclient::Client.new( + join_api_url(api_prefix, api_group), + api_version, + **kubeclient_options + ) + end + + def join_api_url(api_prefix, api_path) + url = URI.parse(api_prefix) + prefix = url.path.sub(%r{/+\z}, '') + + url.path = [prefix, api_path].join("/") + + url.to_s + end + end + end +end diff --git a/lib/gitlab/kubernetes/logger.rb b/lib/gitlab/kubernetes/logger.rb new file mode 100644 index 00000000000..5e59482419b --- /dev/null +++ b/lib/gitlab/kubernetes/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'kubernetes' + end + end + end +end diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index e6ff6160ab9..919f19c86d7 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes class Namespace @@ -10,9 +12,7 @@ module Gitlab def exists? @client.get_namespace(name) - rescue ::Kubeclient::HttpError => ke - raise ke unless ke.error_code == 404 - + rescue ::Kubeclient::ResourceNotFoundError false end diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb index f3842cdf762..81317e532b2 100644 --- a/lib/gitlab/kubernetes/pod.rb +++ b/lib/gitlab/kubernetes/pod.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Kubernetes module Pod diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb new file mode 100644 index 00000000000..cb0cb42d007 --- /dev/null +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class RoleBinding + def initialize(name:, role_name:, namespace:, service_account_name:) + @name = name + @role_name = role_name + @namespace = namespace + @service_account_name = service_account_name + end + + def generate + ::Kubeclient::Resource.new.tap do |resource| + resource.metadata = metadata + resource.roleRef = role_ref + resource.subjects = subjects + end + end + + private + + attr_reader :name, :role_name, :namespace, :service_account_name + + def metadata + { name: name, namespace: namespace } + end + + def role_ref + { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'ClusterRole', + name: role_name + } + end + + def subjects + [ + { + kind: 'ServiceAccount', + name: service_account_name, + namespace: namespace + } + ] + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account.rb b/lib/gitlab/kubernetes/service_account.rb new file mode 100644 index 00000000000..d58fc1c3976 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccount + attr_reader :name, :namespace_name + + def initialize(name, namespace_name) + @name = name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata) + end + + private + + def metadata + { + name: name, + namespace: namespace_name + } + end + end + end +end diff --git a/lib/gitlab/kubernetes/service_account_token.rb b/lib/gitlab/kubernetes/service_account_token.rb new file mode 100644 index 00000000000..2e912b26c09 --- /dev/null +++ b/lib/gitlab/kubernetes/service_account_token.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class ServiceAccountToken + attr_reader :name, :service_account_name, :namespace_name + + def initialize(name, service_account_name, namespace_name) + @name = name + @service_account_name = service_account_name + @namespace_name = namespace_name + end + + def generate + ::Kubeclient::Resource.new(metadata: metadata, type: service_acount_token_type) + end + + private + + # as per https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#to-create-additional-api-tokens + def service_acount_token_type + 'kubernetes.io/service-account-token' + end + + def metadata + { + name: name, + namespace: namespace_name, + annotations: { + "kubernetes.io/service-account.name": service_account_name + } + } + end + end + end +end diff --git a/lib/gitlab/language_data.rb b/lib/gitlab/language_data.rb new file mode 100644 index 00000000000..bfdd7175198 --- /dev/null +++ b/lib/gitlab/language_data.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module LanguageData + EXTENSION_MUTEX = Mutex.new + + class << self + include Gitlab::Utils::StrongMemoize + + def extensions + EXTENSION_MUTEX.synchronize do + strong_memoize(:extensions) do + Set.new.tap do |set| + YAML.load_file(Rails.root.join('vendor', 'languages.yml')).each do |_name, details| + details['extensions']&.each do |ext| + next unless ext.start_with?('.') + + set << ext.downcase + end + end + end + end + end + end + + def clear_extensions! + EXTENSION_MUTEX.synchronize do + clear_memoization(:extensions) + end + end + end + end +end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb new file mode 100644 index 00000000000..7600e60b904 --- /dev/null +++ b/lib/gitlab/language_detection.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + class LanguageDetection + MAX_LANGUAGES = 5 + + def initialize(repository, repository_languages) + @repository = repository + @repository_languages = repository_languages + end + + def languages + detection.keys + end + + def language_color(name) + detection.dig(name, :color) + end + + # Newly detected languages, returned in a structure accepted by + # Gitlab::Database.bulk_insert + def insertions(programming_languages) + lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + + (languages - previous_language_names).map do |new_lang| + { + project_id: @repository.project.id, + share: detection[new_lang][:value], + programming_language_id: lang_to_id[new_lang] + } + end + end + + # updates analyses which records only require updating of their share + def updates + to_update = @repository_languages.select do |lang| + detection.key?(lang.name) && detection[lang.name][:value] != lang.share + end + + to_update.map do |lang| + { programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] } + end + end + + # Returns the ids of the programming languages that do not occur in the detection + # as current repository languages + def deletions + @repository_languages.map do |repo_lang| + next if detection.key?(repo_lang.name) + + repo_lang.programming_language_id + end.compact + end + + private + + def previous_language_names + @previous_language_names ||= @repository_languages.map(&:name) + end + + def detection + @detection ||= + @repository + .languages + .first(MAX_LANGUAGES) + .map { |l| [l[:label], l] } + .to_h + end + end +end diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb index 99594577141..d7a22aa339e 100644 --- a/lib/gitlab/lazy.rb +++ b/lib/gitlab/lazy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # A class that can be wrapped around an expensive method call so it's only # executed when actually needed. diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 2f07fde406c..0b19cf742ed 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class BaseFormatter @@ -10,6 +12,7 @@ module Gitlab @formatter = Gitlab::ImportFormatter.new end + # rubocop: disable CodeReuse/ActiveRecord def create! association = project.public_send(project_association) # rubocop:disable GitlabSecurity/PublicSend @@ -17,6 +20,7 @@ module Gitlab record.attributes = attributes end end + # rubocop: enable CodeReuse/ActiveRecord def url raw_data.url || '' diff --git a/lib/gitlab/legacy_github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 80fe1d67209..1177751457f 100644 --- a/lib/gitlab/legacy_github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class BranchFormatter < BaseFormatter diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index d8ed0ebca9d..bc952147667 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class Client diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index d2c7a8ae9f4..d83cc4f6b3c 100644 --- a/lib/gitlab/legacy_github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class CommentFormatter < BaseFormatter diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index b04d678cf98..c526d31a591 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class Importer @@ -78,8 +80,7 @@ module Gitlab def handle_errors return unless errors.any? - project.ensure_import_state - project.import_state&.update_column(:last_error, { + project.import_state.update_column(:last_error, { message: 'The remote data could not be fully imported.', errors: errors }.to_json) @@ -113,6 +114,7 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| @@ -133,6 +135,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| @@ -193,6 +196,7 @@ module Gitlab issuable.update_attribute(:label_ids, label_ids) end + # rubocop: disable CodeReuse/ActiveRecord def import_comments(issuable_type) resource_type = "#{issuable_type}_comments".to_sym @@ -213,7 +217,9 @@ module Gitlab create_comments(comments) end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| @@ -238,6 +244,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def discard_inserted_comments(comments, last_note) last_note_attrs = nil diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index de55382d3ad..1a0aefbbd62 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class IssuableFormatter < BaseFormatter @@ -55,12 +57,14 @@ module Gitlab end end + # rubocop: disable CodeReuse/ActiveRecord def milestone if raw_data.milestone.present? milestone = MilestoneFormatter.new(project, raw_data.milestone) project.milestones.find_by(milestone.find_condition) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 4c8825ccf19..2f46e2e30d1 100644 --- a/lib/gitlab/legacy_github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class IssueFormatter < IssuableFormatter diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index c3eed12e739..89200e794d8 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class LabelFormatter < BaseFormatter @@ -13,6 +15,7 @@ module Gitlab :labels end + # rubocop: disable CodeReuse/ActiveRecord def create! params = attributes.except(:project) service = ::Labels::FindOrCreateService.new(nil, project, params) @@ -22,6 +25,7 @@ module Gitlab label end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index a565294384d..2fe1b4258d3 100644 --- a/lib/gitlab/legacy_github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class MilestoneFormatter < BaseFormatter diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 3ce245a8050..ca1a1b8e9bd 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class ProjectCreator @@ -35,7 +37,10 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility + visibility_level = repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC + visibility_level = Gitlab::CurrentSettings.default_project_visibility if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level) + + visibility_level end # diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 94c2e99066a..5b847f13d4a 100644 --- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class PullRequestFormatter < IssuableFormatter diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 3ed9d4f76da..8c0c17780ca 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class ReleaseFormatter < BaseFormatter diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 6d8055622f1..ec0e221b1ff 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class UserFormatter @@ -29,6 +31,7 @@ module Gitlab .try(:id) end + # rubocop: disable CodeReuse/ActiveRecord def find_by_external_uid return nil unless id @@ -40,6 +43,7 @@ module Gitlab .first .try(:id) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/legacy_github_import/wiki_formatter.rb b/lib/gitlab/legacy_github_import/wiki_formatter.rb index 27f45875c7c..ea52be5ee0f 100644 --- a/lib/gitlab/legacy_github_import/wiki_formatter.rb +++ b/lib/gitlab/legacy_github_import/wiki_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module LegacyGithubImport class WikiFormatter diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index ead5d566871..26b81847d37 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -1,9 +1,22 @@ +# frozen_string_literal: true + module Gitlab class LfsToken - attr_accessor :actor + module LfsTokenHelper + def user? + actor.is_a?(User) + end + + def actor_name + user? ? actor.username : "lfs+deploy-key-#{actor.id}" + end + end - TOKEN_LENGTH = 50 - EXPIRY_TIME = 1800 + include LfsTokenHelper + + DEFAULT_EXPIRE_TIME = 1800 + + attr_accessor :actor def initialize(actor) @actor = @@ -17,36 +30,108 @@ module Gitlab end end - def token - Gitlab::Redis::SharedState.with do |redis| - token = redis.get(redis_shared_state_key) - token ||= Devise.friendly_token(TOKEN_LENGTH) - redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME) + def token(expire_time: DEFAULT_EXPIRE_TIME) + HMACToken.new(actor).token(expire_time) + end - token - end + def token_valid?(token_to_check) + HMACToken.new(actor).token_valid?(token_to_check) || + LegacyRedisDeviseToken.new(actor).token_valid?(token_to_check) end def deploy_key_pushable?(project) actor.is_a?(DeployKey) && actor.can_push_to?(project) end - def user? - actor.is_a?(User) - end - def type - actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + user? ? :lfs_token : :lfs_deploy_token end - def actor_name - actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + private # rubocop:disable Lint/UselessAccessModifier + + class HMACToken + include LfsTokenHelper + + def initialize(actor) + @actor = actor + end + + def token(expire_time) + hmac_token = JSONWebToken::HMACToken.new(secret) + hmac_token.expire_time = Time.now + expire_time + hmac_token[:data] = { actor: actor_name } + hmac_token.encoded + end + + def token_valid?(token_to_check) + decoded_token = JSONWebToken::HMACToken.decode(token_to_check, secret).first + decoded_token.dig('data', 'actor') == actor_name + rescue JWT::DecodeError + false + end + + private + + attr_reader :actor + + def secret + salt + key + end + + def salt + case actor + when DeployKey, Key + actor.fingerprint.delete(':').first(16) + when User + # Take the last 16 characters as they're more unique than the first 16 + actor.id.to_s + actor.encrypted_password.last(16) + end + end + + def key + # Take 16 characters of attr_encrypted_db_key_base, as that's what the + # cipher needs exactly + Settings.attr_encrypted_db_key_base.first(16) + end end - private + # TODO: LegacyRedisDeviseToken and references need to be removed after + # next released milestone + # + class LegacyRedisDeviseToken + TOKEN_LENGTH = 50 + DEFAULT_EXPIRY_TIME = 1800 * 1000 # 30 mins + + def initialize(actor) + @actor = actor + end - def redis_shared_state_key - "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + def token_valid?(token_to_check) + Devise.secure_compare(stored_token, token_to_check) + end + + def stored_token + Gitlab::Redis::SharedState.with { |redis| redis.get(state_key) } + end + + # This method exists purely to facilitate legacy testing to ensure the + # same redis key is used. + # + def store_new_token(expiry_time_in_ms = DEFAULT_EXPIRY_TIME) + Gitlab::Redis::SharedState.with do |redis| + new_token = Devise.friendly_token(TOKEN_LENGTH) + redis.set(state_key, new_token, px: expiry_time_in_ms) + new_token + end + end + + private + + attr_reader :actor + + def state_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" + end end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index a42e312b5d3..128a5dd8936 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -1,13 +1,23 @@ +# frozen_string_literal: true + module Gitlab class Logger < ::Logger def self.file_name file_name_noext + '.log' end + def self.debug(message) + build.debug(message) + end + def self.error(message) build.error(message) end + def self.warn(message) + build.warn(message) + end + def self.info(message) build.info(message) end @@ -22,7 +32,7 @@ module Gitlab end def self.build - RequestStore[self.cache_key] ||= new(self.full_log_path) + Gitlab::SafeRequestStore[self.cache_key] ||= new(self.full_log_path) end def self.full_log_path diff --git a/lib/gitlab/loop_helpers.rb b/lib/gitlab/loop_helpers.rb new file mode 100644 index 00000000000..3873156a3b0 --- /dev/null +++ b/lib/gitlab/loop_helpers.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module LoopHelpers + ## + # This helper method repeats the same task until it's expired. + # + # Note: ExpiredLoopError does not happen until the given block finished. + # Please do not use this method for heavy or asynchronous operations. + def loop_until(timeout: nil, limit: 1_000_000) + raise ArgumentError unless limit + + start = Time.now + + limit.times do + return true unless yield + + return false if timeout && (Time.now - start) > timeout + end + + false + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 344784c866f..78f2d83c1af 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'yaml' require 'json' require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) @@ -53,7 +55,7 @@ module Gitlab end def config_file - ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__) + ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../config/gitlab.yml', __dir__) end end end diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb new file mode 100644 index 00000000000..7208fe5bbc5 --- /dev/null +++ b/lib/gitlab/manifest_import/manifest.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +# Class to parse manifest file and build a list of repositories for import +# +# <manifest> +# <remote review="https://android-review.googlesource.com/" /> +# <project path="platform-common" name="platform" /> +# <project path="platform/art" name="platform/art" /> +# <project path="platform/device" name="platform/device" /> +# </manifest> +# +# 1. Project path must be uniq and can't be part of other project path. +# For example, you can't have projects with 'foo' and 'foo/bar' paths. +# 2. Remote must be present with review attribute so GitLab knows +# where to fetch source code +module Gitlab + module ManifestImport + class Manifest + attr_reader :parsed_xml, :errors + + def initialize(file) + @parsed_xml = Nokogiri::XML(file) { |config| config.strict } + @errors = [] + rescue Nokogiri::XML::SyntaxError + @errors = ['The uploaded file is not a valid XML file.'] + end + + def projects + raw_projects.each_with_index.map do |project, i| + { + id: i, + name: project['name'], + path: project['path'], + url: repository_url(project['name']) + } + end + end + + def valid? + return false if @errors.any? + + unless validate_remote + @errors << 'Make sure a <remote> tag is present and is valid.' + end + + unless validate_projects + @errors << 'Make sure every <project> tag has name and path attributes.' + end + + @errors.empty? + end + + private + + def validate_remote + remote.present? && URI.parse(remote).host + rescue URI::Error + false + end + + def validate_projects + raw_projects.all? do |project| + project['name'] && project['path'] + end + end + + def repository_url(name) + Gitlab::Utils.append_path(remote, name) + end + + def remote + return @remote if defined?(@remote) + + remote_tag = parsed_xml.css('manifest > remote').first + @remote = remote_tag['review'] if remote_tag + end + + def raw_projects + @raw_projects ||= parsed_xml.css('manifest > project') + end + end + end +end diff --git a/lib/gitlab/manifest_import/project_creator.rb b/lib/gitlab/manifest_import/project_creator.rb new file mode 100644 index 00000000000..837d65e5f7c --- /dev/null +++ b/lib/gitlab/manifest_import/project_creator.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module ManifestImport + class ProjectCreator + attr_reader :repository, :destination, :current_user + + def initialize(repository, destination, current_user) + @repository = repository + @destination = destination + @current_user = current_user + end + + def execute + group_full_path, _, project_path = repository[:path].rpartition('/') + group_full_path = File.join(destination.full_path, group_full_path) if destination + group = create_group_with_parents(group_full_path) + + params = { + import_url: repository[:url], + import_type: 'manifest', + namespace_id: group.id, + path: project_path, + name: project_path, + visibility_level: destination.visibility_level + } + + Projects::CreateService.new(current_user, params).execute + end + + private + + def create_group_with_parents(full_path) + params = { + group_path: full_path, + visibility_level: destination.visibility_level + } + + Groups::NestedCreateService.new(current_user, params).execute + end + end + end +end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index 49285e35251..d419fa66e57 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -1,11 +1,14 @@ +# frozen_string_literal: true + module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze - ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze - OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze + OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS + PLAIN_FILENAMES = %w[readme index].freeze # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -41,7 +44,7 @@ module Gitlab # # Returns boolean def plain?(filename) - extension(filename) == 'txt' || filename.casecmp('readme').zero? + extension(filename) == 'txt' || plain_filename?(filename) end def previewable?(filename) @@ -53,5 +56,9 @@ module Gitlab def extension(filename) File.extname(filename).downcase.delete('.') end + + def plain_filename?(filename) + PLAIN_FILENAMES.include?(filename.downcase) + end end end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 7d63ca5627d..61ed20ad623 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics include Gitlab::Metrics::InfluxDb diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index 5919ebb1493..fe1722b1095 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics class BackgroundTransaction < Transaction diff --git a/lib/gitlab/metrics/delta.rb b/lib/gitlab/metrics/delta.rb index bcf28eed84d..ab2d9e46390 100644 --- a/lib/gitlab/metrics/delta.rb +++ b/lib/gitlab/metrics/delta.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Class for calculating the difference between two numeric values. diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 66f30e3b397..0b04340fbb5 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module InfluxDb @@ -86,7 +88,7 @@ module Gitlab # Example: # # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) + # UserFinder.new(some_username).find_by_username # end # # name - The name of the field to store the execution time in. @@ -145,9 +147,7 @@ module Gitlab # # See `Gitlab::Metrics::Transaction#add_event` for more details. def add_event(*args) - trans = current_transaction - - trans&.add_event(*args) + current_transaction&.add_event(*args) end # Returns the prefix to use for the name of a series. @@ -162,7 +162,6 @@ module Gitlab # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. - # rubocop:disable Gitlab/ModuleWithInstanceVariables def pool if influx_metrics_enabled? if @pool.nil? @@ -180,7 +179,6 @@ module Gitlab @pool end end - # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 023e9963493..651e241362c 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Module for instrumenting methods. diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index b11520a79bb..85438011cb9 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -1,4 +1,4 @@ -# rubocop:disable Style/ClassVars +# frozen_string_literal: true module Gitlab module Metrics diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index f79eb0cd1bf..447d03bfca4 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable Style/ClassVars module Gitlab diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb index 70e122d4e15..8e6ceb74c09 100644 --- a/lib/gitlab/metrics/methods/metric_options.rb +++ b/lib/gitlab/metrics/methods/metric_options.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Methods diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index bd0afe53c51..9e4d70a71ff 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb index aabada5c21a..7dbd2a1f8e3 100644 --- a/lib/gitlab/metrics/null_metric.rb +++ b/lib/gitlab/metrics/null_metric.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Mocks ::Prometheus::Client::Metric and all derived metrics diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index d41a855bff1..cab1edab48f 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'prometheus/client' module Gitlab diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 2d45765df3f..9aa97515961 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Rack middleware for tracking Rails and Grape requests. diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 0dc19f31d03..74c956ab5af 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics class RequestsRackMiddleware diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 37f90c4673d..6a062e93f0f 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' module Gitlab diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5a0f7f28fc8..c4c38b23a55 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Samplers @@ -16,12 +18,6 @@ module Gitlab @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) - - if Gitlab::Metrics.mri? - require 'allocations' - - Allocations.start - end end def sample diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4e1ea62351f..232a58a7d69 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'prometheus/client/support/unicorn' module Gitlab @@ -20,39 +22,29 @@ module Gitlab {} end - def initialize(interval) - super(interval) - - if Metrics.mri? - require 'allocations' - - Allocations.start - end - end - def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -60,11 +52,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index ea325651fbb..4c4ec026823 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Samplers diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index 47b4af5d649..56e106b9612 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'webrick' require 'prometheus/client/rack/exporter' diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index df4bdf16847..0b4485feea9 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Sidekiq middleware for tracking jobs. diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index b600e8a2a50..c068f8017fd 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Subscribers diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 4b3e8d0a6a0..a02dd850582 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Subscribers @@ -6,9 +8,15 @@ module Gitlab include Gitlab::Metrics::Methods attach_to :active_record + IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze + def sql(event) return unless current_transaction + payload = event.payload + + return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) @@ -20,7 +28,7 @@ module Gitlab define_histogram :gitlab_sql_duration_seconds do docstring 'SQL time' base_labels Transaction::BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 250897a79c2..f633e1a9d7c 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics module Subscribers diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index e60e245cf89..426496855e3 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Module for gathering system/process statistics such as the memory usage. diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index f3e48083c19..468d7cb56fc 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Metrics # Class for storing metrics information of a single transaction. @@ -140,7 +142,7 @@ module Gitlab define_histogram :gitlab_transaction_duration_seconds do docstring 'Transaction duration' base_labels BASE_LABELS - buckets [0.001, 0.01, 0.1, 1.0, 10.0] + buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end define_histogram :gitlab_transaction_allocated_memory_bytes do diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 3799aaebf1c..b2a43d46fb2 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -1,8 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Metrics class WebTransaction < Transaction CONTROLLER_KEY = 'action_controller.instance'.freeze ENDPOINT_KEY = 'api.endpoint'.freeze + ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) def initialize(env) super() @@ -32,10 +35,14 @@ module Gitlab # Devise exposes a method called "request_format" that does the below. # However, this method is not available to all controllers (e.g. certain # Doorkeeper controllers). As such we use the underlying code directly. - suffix = controller.request.format.try(:ref) + suffix = controller.request.format.try(:ref).to_s - if suffix && suffix != :html - action += ".#{suffix}" + # Sometimes the request format is set to silly data such as + # "application/xrds+xml" or actual URLs. To prevent such values from + # increasing the cardinality of our metrics, we limit the number of + # possible suffixes. + if suffix && ALLOWED_SUFFIXES.include?(suffix) + action = "#{action}.#{suffix}" end { controller: controller.class.name, action: action } diff --git a/lib/gitlab/middleware/basic_health_check.rb b/lib/gitlab/middleware/basic_health_check.rb new file mode 100644 index 00000000000..acf8c301b8f --- /dev/null +++ b/lib/gitlab/middleware/basic_health_check.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# This middleware provides a health check that does not hit the database. Its purpose +# is to notify the prober that the application server is handling requests, but a 200 +# response does not signify that the database or other services are ready. +# +# See https://thisdata.com/blog/making-a-rails-health-check-that-doesnt-hit-the-database/ for +# more details. + +module Gitlab + module Middleware + class BasicHealthCheck + # This can't be frozen because Rails::Rack::Logger wraps the body + # rubocop:disable Style/MutableConstant + OK_RESPONSE = [200, { 'Content-Type' => 'text/plain' }, ["GitLab OK"]] + EMPTY_RESPONSE = [404, { 'Content-Type' => 'text/plain' }, [""]] + # rubocop:enable Style/MutableConstant + HEALTH_PATH = '/-/health' + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless env['PATH_INFO'] == HEALTH_PATH + + request = ActionDispatch::Request.new(env) + + return OK_RESPONSE if client_ip_whitelisted?(request) + + EMPTY_RESPONSE + end + + def client_ip_whitelisted?(request) + ip_whitelist.any? { |e| e.include?(request.ip) } + end + + def ip_whitelist + @ip_whitelist ||= Settings.monitoring.ip_whitelist.map(&IPAddr.method(:new)) + end + end + end +end diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb new file mode 100644 index 00000000000..80dddc41c12 --- /dev/null +++ b/lib/gitlab/middleware/correlation_id.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# A dumb middleware that steals correlation id +# and sets it as a global context for the request +module Gitlab + module Middleware + class CorrelationId + include ActionView::Helpers::TagHelper + + def initialize(app) + @app = app + end + + def call(env) + ::Gitlab::CorrelationId.use_id(correlation_id(env)) do + @app.call(env) + end + end + + private + + def correlation_id(env) + request(env).request_id + end + + def request(env) + ActionDispatch::Request.new(env) + end + end + end +end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 1fd8f147b44..f9efef38825 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -1,9 +1,12 @@ +# frozen_string_literal: true + # A dumb middleware that returns a Go HTML document if the go-get=1 query string # is used irrespective if the namespace/project exists module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper + include ActionController::HttpAuthentication::Basic PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -12,7 +15,7 @@ module Gitlab end def call(env) - request = Rack::Request.new(env) + request = ActionDispatch::Request.new(env) render_go_doc(request) || @app.call(env) end @@ -38,7 +41,7 @@ module Gitlab def go_body(path) config = Gitlab.config - project_url = URI.join(config.gitlab.url, path) + project_url = Gitlab::Utils.append_path(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' @@ -108,21 +111,23 @@ module Gitlab def project_for_paths(paths, request) project = Project.where_full_path_in(paths).first - return unless Ability.allowed?(current_user(request), :read_project, project) + return unless Ability.allowed?(current_user(request, project), :read_project, project) project end - def current_user(request) - authenticator = Gitlab::Auth::RequestAuthenticator.new(request) - user = authenticator.find_user_from_access_token || authenticator.find_user_from_warden + def current_user(request, project) + return unless has_basic_credentials?(request) + + login, password = user_name_and_password(request) + auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) + return unless auth_result.success? - return unless user&.can?(:access_api) + return unless auth_result.actor&.can?(:access_git) - # Right now, the `api` scope is the only one that should be able to determine private project existence. - return unless authenticator.valid_access_token?(scopes: [:api]) + return unless auth_result.authentication_abilities.include?(:read_project) - user + auth_result.actor end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a5f5d719cc1..433151b80e7 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitlab::Middleware::Multipart - a Rack::Multipart replacement # # Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby @@ -30,7 +32,7 @@ module Gitlab class Handler def initialize(env, message) - @request = Rack::Request.new(env) + @request = ActionDispatch::Request.new(env) @rewritten_fields = message['rewritten_fields'] @open_files = [] end @@ -42,10 +44,10 @@ module Gitlab key, value = parsed_field.first if value.nil? - value = open_file(tmp_path, @request.params["#{key}.name"]) + value = open_file(@request.params, key) @open_files << value else - value = decorate_params_value(value, @request.params[key], tmp_path) + value = decorate_params_value(value, @request.params[key]) end @request.update_param(key, value) @@ -57,7 +59,7 @@ module Gitlab end # This function calls itself recursively - def decorate_params_value(path_hash, value_hash, tmp_path) + def decorate_params_value(path_hash, value_hash) unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end @@ -70,19 +72,25 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(tmp_path, value_hash.dig(path_key, '.name')) + value_hash[path_key] = open_file(value_hash.dig(path_key), '') @open_files << value_hash[path_key] value_hash when Hash - decorate_params_value(path_value, value_hash[path_key], tmp_path) + decorate_params_value(path_value, value_hash[path_key]) value_hash else raise "unexpected path value: #{path_value.inspect}" end end - def open_file(path, name) - ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream') + def open_file(params, key) + allowed_paths = [ + ::FileUploader.root, + Gitlab.config.uploads.storage_path, + File.join(Rails.root, 'public/uploads/tmp') + ] + + ::UploadedFile.from_params(params, key, allowed_paths) end end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index bc70b2459ef..96c6a0a7d28 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This Rack middleware is intended to measure the latency between # gitlab-workhorse forwarding a request to the Rails application and the # time this middleware is reached. diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 7f63e39b3aa..83c52a6c6e0 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Middleware class ReadOnly diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 45b644e6510..817db12ac55 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -1,11 +1,23 @@ +# frozen_string_literal: true + module Gitlab module Middleware class ReadOnly class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + WHITELISTED_GIT_ROUTES = { + 'projects/git_http' => %w{git_upload_pack git_receive_pack} + }.freeze + + WHITELISTED_GIT_LFS_ROUTES = { + 'projects/lfs_api' => %w{batch}, + 'projects/lfs_locks_api' => %w{verify create unlock} + }.freeze + def initialize(app, env) @app = app @env = env @@ -36,7 +48,7 @@ module Gitlab end def json_request? - request.media_type == APPLICATION_JSON + APPLICATION_JSON_TYPES.include?(request.media_type) end def rack_flash @@ -48,7 +60,7 @@ module Gitlab end def request - @env['rack.request'] ||= Rack::Request.new(@env) + @env['actionpack.request'] ||= ActionDispatch::Request.new(@env) end def last_visited_url @@ -59,26 +71,40 @@ module Gitlab @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} end - def whitelisted_routes - grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + def relative_url + File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/') end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') + # Overridden in EE module + def whitelisted_routes + grack_route? || internal_route? || lfs_route? || sidekiq_route? end - def grack_route + def grack_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') + return false unless + request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end - def lfs_route + def internal_route? + ReadOnly.internal_routes.any? { |path| request.path.include?(path) } + end + + def lfs_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') + unless request.path.end_with?('/info/lfs/objects/batch', + '/info/lfs/locks', '/info/lfs/locks/verify') || + %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) + return false + end + + WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + def sidekiq_route? + request.path.start_with?("#{relative_url}/admin/sidekiq") end end end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb index bfe8e113b5e..849cf8f759b 100644 --- a/lib/gitlab/middleware/release_env.rb +++ b/lib/gitlab/middleware/release_env.rb @@ -1,4 +1,7 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module Middleware # Some of middleware would hold env for no good reason even after the # request had already been processed, and we could not garbage collect diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index aa1e9dc0fdb..972fed2134c 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Middleware class Static < ActionDispatch::Static diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index fd5de73c526..5375077d7dc 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class MultiCollectionPaginator attr_reader :first_collection, :second_collection, :per_page @@ -53,6 +55,7 @@ module Gitlab @first_collection_page_count = first_collection_page.total_pages end + # rubocop: disable CodeReuse/ActiveRecord def first_collection_last_page_size return @first_collection_last_page_size if defined?(@first_collection_last_page_size) @@ -60,5 +63,6 @@ module Gitlab .except(:select) .size end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/namespace_sanitizer.rb b/lib/gitlab/namespace_sanitizer.rb new file mode 100644 index 00000000000..d755bbbcaf9 --- /dev/null +++ b/lib/gitlab/namespace_sanitizer.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class NamespaceSanitizer + def self.sanitize(namespace) + namespace.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') + end + end +end diff --git a/lib/gitlab/null_request_store.rb b/lib/gitlab/null_request_store.rb new file mode 100644 index 00000000000..8db331dcb9f --- /dev/null +++ b/lib/gitlab/null_request_store.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Used by Gitlab::SafeRequestStore +module Gitlab + # The methods `begin!`, `clear!`, and `end!` are not defined because they + # should only be called directly on `RequestStore`. + class NullRequestStore + def store + {} + end + + def active? + end + + def read(key) + end + + def [](key) + end + + def write(key, value) + value + end + + def []=(key, value) + value + end + + def exist?(key) + false + end + + def fetch(key, &block) + yield + end + + def delete(key, &block) + yield(key) if block_given? + end + end +end diff --git a/lib/gitlab/group_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 42ded7c286f..f2772c733c7 100644 --- a/lib/gitlab/group_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -1,14 +1,16 @@ +# frozen_string_literal: true + module Gitlab - # Retrieving of parent or child groups based on a base ActiveRecord relation. + # Retrieving of parent or child objects based on a base ActiveRecord relation. # # This class uses recursive CTEs and as a result will only work on PostgreSQL. - class GroupHierarchy + class ObjectHierarchy attr_reader :ancestors_base, :descendants_base, :model # ancestors_base - An instance of ActiveRecord::Relation for which to - # get parent groups. + # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to - # get child groups. If omitted, ancestors_base is used. + # get child objects. If omitted, ancestors_base is used. def initialize(ancestors_base, descendants_base = ancestors_base) raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model @@ -19,9 +21,11 @@ module Gitlab # Returns the set of descendants of a given relation, but excluding the given # relation + # rubocop: disable CodeReuse/ActiveRecord def descendants base_and_descendants.where.not(id: descendants_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord # Returns the set of ancestors of a given relation, but excluding the given # relation @@ -29,32 +33,45 @@ module Gitlab # Passing an `upto` will stop the recursion once the specified parent_id is # reached. So all ancestors *lower* than the specified ancestor will be # included. - def ancestors(upto: nil) - base_and_ancestors(upto: upto).where.not(id: ancestors_base.select(:id)) + # rubocop: disable CodeReuse/ActiveRecord + def ancestors(upto: nil, hierarchy_order: nil) + base_and_ancestors(upto: upto, hierarchy_order: hierarchy_order).where.not(id: ancestors_base.select(:id)) end + # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the ancestors_base set of groups + # Returns a relation that includes the ancestors_base set of objects # and all their ancestors (recursively). # # Passing an `upto` will stop the recursion once the specified parent_id is # reached. So all ancestors *lower* than the specified acestor will be # included. - def base_and_ancestors(upto: nil) - return ancestors_base unless Group.supports_nested_groups? - - read_only(base_and_ancestors_cte(upto).apply_to(model.all)) + # + # Passing a `hierarchy_order` with either `:asc` or `:desc` will cause the + # recursive query order from most nested object to root or from the root + # ancestor to most nested object respectively. This uses a `depth` column + # where `1` is defined as the depth for the base and increment as we go up + # each parent. + # rubocop: disable CodeReuse/ActiveRecord + def base_and_ancestors(upto: nil, hierarchy_order: nil) + return ancestors_base unless hierarchy_supported? + + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) + recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + + read_only(recursive_query) end + # rubocop: enable CodeReuse/ActiveRecord - # Returns a relation that includes the descendants_base set of groups + # Returns a relation that includes the descendants_base set of objects # and all their descendants (recursively). def base_and_descendants - return descendants_base unless Group.supports_nested_groups? + return descendants_base unless hierarchy_supported? read_only(base_and_descendants_cte.apply_to(model.all)) end - # Returns a relation that includes the base groups, their ancestors, - # and the descendants of the base groups. + # Returns a relation that includes the base objects, their ancestors, + # and the descendants of the base objects. # # The resulting query will roughly look like the following: # @@ -74,46 +91,61 @@ module Gitlab # Using this approach allows us to further add criteria to the relation with # Rails thinking it's selecting data the usual way. # - # If nested groups are not supported, ancestors_base is returned. - def all_groups - return ancestors_base unless Group.supports_nested_groups? + # If nested objects are not supported, ancestors_base is returned. + # rubocop: disable CodeReuse/ActiveRecord + def all_objects + return ancestors_base unless hierarchy_supported? ancestors = base_and_ancestors_cte descendants = base_and_descendants_cte - ancestors_table = ancestors.alias_to(groups_table) - descendants_table = descendants.alias_to(groups_table) - - union = SQL::Union.new([model.unscoped.from(ancestors_table), - model.unscoped.from(descendants_table)]) + ancestors_table = ancestors.alias_to(objects_table) + descendants_table = descendants.alias_to(objects_table) relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) - .from("(#{union.to_sql}) #{model.table_name}") + .from_union([ + model.unscoped.from(ancestors_table), + model.unscoped.from(descendants_table) + ]) read_only(relation) end + # rubocop: enable CodeReuse/ActiveRecord private - def base_and_ancestors_cte(stop_id = nil) + def hierarchy_supported? + Gitlab::Database.postgresql? + end + + # rubocop: disable CodeReuse/ActiveRecord + def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) + depth_column = :depth - cte << ancestors_base.except(:order) + base_query = ancestors_base.except(:order) + base_query = base_query.select("1 as #{depth_column}", objects_table[Arel.star]) if hierarchy_order + + cte << base_query # Recursively get all the ancestors of the base set. parent_query = model - .from([groups_table, cte.table]) - .where(groups_table[:id].eq(cte.table[:parent_id])) + .from([objects_table, cte.table]) + .where(objects_table[:id].eq(cte.table[:parent_id])) .except(:order) + + parent_query = parent_query.select(cte.table[depth_column] + 1, objects_table[Arel.star]) if hierarchy_order parent_query = parent_query.where(cte.table[:parent_id].not_eq(stop_id)) if stop_id cte << parent_query cte end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants_cte cte = SQL::RecursiveCTE.new(:base_and_descendants) @@ -121,14 +153,15 @@ module Gitlab # Recursively get all the descendants of the base set. cte << model - .from([groups_table, cte.table]) - .where(groups_table[:parent_id].eq(cte.table[:id])) + .from([objects_table, cte.table]) + .where(objects_table[:parent_id].eq(cte.table[:id])) .except(:order) cte end + # rubocop: enable CodeReuse/ActiveRecord - def groups_table + def objects_table model.arel_table end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 35ed3a5ac05..e0ac9eec1f2 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class OmniauthInitializer def initialize(devise_config) @@ -6,13 +8,16 @@ module Gitlab def execute(providers) providers.each do |provider| - add_provider(provider['name'].to_sym, *arguments_for(provider)) + name = provider['name'].to_sym + + add_provider_to_devise(name, *arguments_for(provider)) + setup_provider(name) end end private - def add_provider(*args) + def add_provider_to_devise(*args) @devise_config.omniauth(*args) end @@ -71,5 +76,23 @@ module Gitlab end end end + + def omniauth_customized_providers + @omniauth_customized_providers ||= build_omniauth_customized_providers + end + + # We override this in EE + def build_omniauth_customized_providers + %i[bitbucket jwt] + end + + def setup_provider(provider) + case provider + when :kerberos + require 'omniauth-kerberos' + when *omniauth_customized_providers + require_dependency "omni_auth/strategies/#{provider}" + end + end end end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index d09bce642b0..ce4ba9f752b 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module OptimisticLocking module_function diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index fc3f21233dd..bc467486eee 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 22332474945..1d3200aa099 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute. # @@ -26,6 +28,7 @@ module Gitlab @filename = filename end + # rubocop: disable CodeReuse/ActiveRecord def rotate!(old_key:, new_key:) old_key ||= Gitlab::Application.secrets.otp_key_base @@ -47,7 +50,9 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def rollback! ActiveRecord::Base.transaction do CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| @@ -55,6 +60,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 981ef8faa9a..16df0700b08 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze diff --git a/lib/gitlab/pages_client.rb b/lib/gitlab/pages_client.rb index 7b358a3bd1b..d74fdba2241 100644 --- a/lib/gitlab/pages_client.rb +++ b/lib/gitlab/pages_client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class PagesClient class << self @@ -101,7 +103,7 @@ module Gitlab end def write_token(new_token) - Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| + Tempfile.open(File.basename(token_path), File.dirname(token_path), encoding: 'ascii-8bit') do |f| f.write(new_token) f.close File.link(f.path, token_path) diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index fb215f27cbd..a70dc826f97 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class PagesTransfer < ProjectTransfer def root_dir diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb new file mode 100644 index 00000000000..b00244a6e04 --- /dev/null +++ b/lib/gitlab/patch/draw_route.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# We're patching `ActionDispatch::Routing::Mapper` in +# config/initializers/routing_draw.rb +module Gitlab + module Patch + module DrawRoute + RoutesNotFound = Class.new(StandardError) + + def draw(routes_name) + drawn_any = draw_ce(routes_name) | draw_ee(routes_name) + + drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) + end + + def draw_ce(routes_name) + draw_route(route_path("config/routes/#{routes_name}.rb")) + end + + def draw_ee(_) + true + end + + def route_path(routes_name) + Rails.root.join(routes_name) + end + + def draw_route(path) + if File.exist?(path) + instance_eval(File.read(path)) + true + else + false + end + end + end + end +end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb new file mode 100644 index 00000000000..a9f6cfb19cb --- /dev/null +++ b/lib/gitlab/patch/prependable.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# We're patching `ActiveSupport::Concern` in +# config/initializers/0_as_concern.rb +# +# We want to patch `ActiveSupport::Concern` for two reasons: +# 1. Allow defining class methods via: `class_methods` method +# 2. Allow `prepended do; end` work like `included do; end` +# If we don't need anything above, we don't need this patch nor the concern! + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Patch + module Prependable + class MultiplePrependedBlocks < StandardError + def initialize + super "Cannot define multiple 'prepended' blocks for a Concern" + end + end + + def prepend_features(base) + return false if prepended?(base) + + super + + if const_defined?(:ClassMethods) + klass_methods = const_get(:ClassMethods) + base.singleton_class.prepend klass_methods + base.instance_variable_set(:@_prepended_class_methods, klass_methods) + end + + if instance_variable_defined?(:@_prepended_block) + base.class_eval(&@_prepended_block) + end + + true + end + + def class_methods + super + + if instance_variable_defined?(:@_prepended_class_methods) + const_get(:ClassMethods).prepend @_prepended_class_methods + end + end + + def prepended(base = nil, &block) + if base.nil? + raise MultiplePrependedBlocks if + instance_variable_defined?(:@_prepended_block) + + @_prepended_block = block + else + super + end + end + + def prepended?(base) + index = base.ancestors.index(base) + + base.ancestors[0...index].index(self) + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 4dc38aae61e..3c888be0710 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module PathRegex extend self @@ -31,6 +33,7 @@ module Gitlab deploy.html explore favicon.ico + favicon.png files groups health_check @@ -38,7 +41,7 @@ module Gitlab import invites jwt - koding + login notification_settings oauth profile @@ -122,7 +125,8 @@ module Gitlab # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + PATH_START_CHAR = '[a-zA-Z0-9_\.]'.freeze + PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]*'.freeze NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze @@ -233,7 +237,7 @@ module Gitlab def single_line_regexp(regex) # Turns a multiline extended regexp into a single line one, - # beacuse `rake routes` breaks on multiline regexes. + # because `rake routes` breaks on multiline regexes. Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 92a308a12dc..4b0c7b5c7f8 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module PerformanceBar ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze @@ -15,6 +17,7 @@ module Gitlab Gitlab::CurrentSettings.performance_bar_allowed_group_id end + # rubocop: disable CodeReuse/ActiveRecord def self.allowed_user_ids Rails.cache.fetch(ALLOWED_USER_IDS_KEY, expires_in: EXPIRY_TIME) do group = Group.find_by_id(allowed_group_id) @@ -26,6 +29,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord def self.expire_allowed_user_ids_cache Rails.cache.delete(ALLOWED_USER_IDS_KEY) diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index f2825db59ae..ac392432427 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Inspired by https://github.com/peek/peek-pg/blob/master/lib/peek/views/pg.rb # PEEK_DB_CLIENT is a constant set in config/initializers/peek.rb module Gitlab @@ -23,7 +25,7 @@ module Gitlab end subscribe('sql.active_record') do |_, start, finish, _, data| - if RequestStore.active? && RequestStore.store[:peek_enabled] + if Gitlab::SafeRequestStore.store[:peek_enabled] # data[:cached] is only available starting from Rails 5.1.0 # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 # Before that, data[:name] was set to 'CACHE' diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/plugin.rb index 0d1cb16b378..23353f36025 100644 --- a/lib/gitlab/plugin.rb +++ b/lib/gitlab/plugin.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Plugin def self.files diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/plugin_logger.rb index c4f6ec3e21d..df3bd56fd2f 100644 --- a/lib/gitlab/plugin_logger.rb +++ b/lib/gitlab/plugin_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class PluginLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index fe4bdfe3831..0f69990df63 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class PollingInterval HEADER_NAME = 'Poll-Interval'.freeze diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index b9832a724c4..7fa00d0c68c 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'fileutils' require 'open3' @@ -11,7 +13,7 @@ module Gitlab def popen(cmd, path = nil, vars = {}, &block) result = popen_with_detail(cmd, path, vars, &block) - [result.stdout << result.stderr, result.status&.exitstatus] + ["#{result.stdout}#{result.stderr}", result.status&.exitstatus] end # Returns Result @@ -34,11 +36,16 @@ module Gitlab start = Time.now Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 + # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 + out_reader = Thread.new { stdout.read } + err_reader = Thread.new { stderr.read } + yield(stdin) if block_given? stdin.close - cmd_stdout = stdout.read - cmd_stderr = stderr.read + cmd_stdout = out_reader.value + cmd_stderr = err_reader.value cmd_status = wait_thr.value end diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb index f44035a48bb..cd9ad270cd8 100644 --- a/lib/gitlab/popen/runner.rb +++ b/lib/gitlab/popen/runner.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Popen class Runner diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb new file mode 100644 index 00000000000..536fc9dae3a --- /dev/null +++ b/lib/gitlab/private_commit_email.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module PrivateCommitEmail + TOKEN = "_private".freeze + + class << self + def regex + hostname_regexp = Regexp.escape(Gitlab::CurrentSettings.current_application_settings.commit_email_hostname) + + /\A(?<id>([0-9]+))\-([^@]+)@#{hostname_regexp}\z/ + end + + def user_id_for_email(email) + match = email&.match(regex) + return unless match + + match[:id].to_i + end + + def user_ids_for_emails(emails) + emails.map { |email| user_id_for_email(email) }.compact.uniq + end + + def for_user(user) + hostname = Gitlab::CurrentSettings.current_application_settings.commit_email_hostname + + "#{user.id}-#{user.username}@#{hostname}" + end + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 18540e64d4c..93a9fcf1591 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -1,4 +1,6 @@ # coding: utf-8 +# frozen_string_literal: true + module Gitlab module Profiler FILTERED_STRING = '[FILTERED]'.freeze @@ -11,6 +13,7 @@ module Gitlab lib/gitlab/etag_caching/ lib/gitlab/metrics/ lib/gitlab/middleware/ + ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ lib/gitlab/request_profiler/ lib/gitlab/profiler.rb @@ -43,12 +46,11 @@ module Gitlab headers['Content-Type'] = 'application/json' end - if user - private_token ||= user.personal_access_tokens.active.pluck(:token).first - raise 'Your user must have a personal_access_token' unless private_token + if private_token + headers['Private-Token'] = private_token + user = nil # private_token overrides user end - headers['Private-Token'] = private_token if private_token logger = create_custom_logger(logger, private_token: private_token) RequestStore.begin! @@ -66,7 +68,9 @@ module Gitlab app.get('/api/v4/users') result = with_custom_logger(logger) do - RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + with_user(user) do + RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + end end RequestStore.end! @@ -98,11 +102,7 @@ module Gitlab super - backtrace = Rails.backtrace_cleaner.clean(caller) - - backtrace.each do |caller_line| - next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) - + Gitlab::Profiler.clean_backtrace(caller).each do |caller_line| stripped_caller_line = caller_line.sub("#{Rails.root}/", '') super(" ↳ #{stripped_caller_line}") @@ -112,6 +112,12 @@ module Gitlab end end + def self.clean_backtrace(backtrace) + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| + line.match(Regexp.union(IGNORE_BACKTRACES)) + end + end + def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging original_activerecord_logger = ActiveRecord::Base.logger @@ -123,15 +129,32 @@ module Gitlab ActionController::Base.logger = logger end - result = yield + yield.tap do + ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging + ActiveRecord::Base.logger = original_activerecord_logger + ActionController::Base.logger = original_actioncontroller_logger + end + end - ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging - ActiveRecord::Base.logger = original_activerecord_logger - ActionController::Base.logger = original_actioncontroller_logger + def self.with_user(user) + if user + API::Helpers::CommonHelpers.send(:define_method, :find_current_user!) { user } # rubocop:disable GitlabSecurity/PublicSend + ApplicationController.send(:define_method, :current_user) { user } # rubocop:disable GitlabSecurity/PublicSend + ApplicationController.send(:define_method, :authenticate_user!) { } # rubocop:disable GitlabSecurity/PublicSend + end - result + yield.tap do + remove_method(API::Helpers::CommonHelpers, :find_current_user!) + remove_method(ApplicationController, :current_user) + remove_method(ApplicationController, :authenticate_user!) + end end + def self.remove_method(klass, meth) + klass.send(:remove_method, meth) if klass.instance_methods(false).include?(meth) # rubocop:disable GitlabSecurity/PublicSend + end + + # rubocop: disable CodeReuse/ActiveRecord def self.log_load_times_by_model(logger) return unless logger.respond_to?(:load_times_by_model) @@ -143,5 +166,12 @@ module Gitlab logger.info("#{model} total (#{query_count}): #{time.round(2)}ms") end end + # rubocop: enable CodeReuse/ActiveRecord + + def self.print_by_total_time(result, options = {}) + default_options = { sort_method: :total_time } + + Gitlab::Profiler::TotalTimeFlatPrinter.new(result).print(STDOUT, default_options.merge(options)) + end end end diff --git a/lib/gitlab/profiler/total_time_flat_printer.rb b/lib/gitlab/profiler/total_time_flat_printer.rb new file mode 100644 index 00000000000..2c105d2722b --- /dev/null +++ b/lib/gitlab/profiler/total_time_flat_printer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Profiler + class TotalTimeFlatPrinter < RubyProf::FlatPrinter + def max_percent + @options[:max_percent] || 100 + end + + # Copied from: + # <https://github.com/ruby-prof/ruby-prof/blob/master/lib/ruby-prof/printers/flat_printer.rb> + # + # The changes are just to filter by total time, not self time, and add a + # max_percent option as well. + def print_methods(thread) + total_time = thread.total_time + methods = thread.methods.sort_by(&sort_method).reverse + + sum = 0 + methods.each do |method| + total_percent = (method.total_time / total_time) * 100 + next if total_percent < min_percent + next if total_percent > max_percent + + sum += method.self_time + + @output << "%6.2f %9.3f %9.3f %9.3f %9.3f %8d %s%s\n" % [ + method.self_time / total_time * 100, # %self + method.total_time, # total + method.self_time, # self + method.wait_time, # wait + method.children_time, # children + method.called, # calls + method.recursive? ? "*" : " ", # cycle + method_name(method) # name + ] + end + end + end + end +end diff --git a/lib/gitlab/project_authorizations/with_nested_groups.rb b/lib/gitlab/project_authorizations/with_nested_groups.rb index 15b8beacf60..2372a316ab0 100644 --- a/lib/gitlab/project_authorizations/with_nested_groups.rb +++ b/lib/gitlab/project_authorizations/with_nested_groups.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ProjectAuthorizations # Calculating new project authorizations when supporting nested groups. @@ -24,7 +26,7 @@ module Gitlab user.projects.select_for_project_authorization, # The personal projects of the user. - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects that belong directly to any of the groups the user has # access to. @@ -49,13 +51,11 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped .with .recursive(cte.to_arel) - .select_from_union(union) + .select_from_union(relations) end private diff --git a/lib/gitlab/project_authorizations/without_nested_groups.rb b/lib/gitlab/project_authorizations/without_nested_groups.rb index ad87540e6c2..50b41b17649 100644 --- a/lib/gitlab/project_authorizations/without_nested_groups.rb +++ b/lib/gitlab/project_authorizations/without_nested_groups.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ProjectAuthorizations # Calculating new project authorizations when not supporting nested groups. @@ -15,7 +17,7 @@ module Gitlab user.projects.select_for_project_authorization, # Personal projects - user.personal_projects.select_as_master_for_project_authorization, + user.personal_projects.select_as_maintainer_for_project_authorization, # Projects of groups the user is a member of user.groups_projects.select_for_project_authorization, @@ -24,11 +26,9 @@ module Gitlab user.groups.joins(:shared_projects).select_for_project_authorization ] - union = Gitlab::SQL::Union.new(relations) - ProjectAuthorization .unscoped - .select_from_union(union) + .select_from_union(relations) end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 2e9b6e302f5..a68f8801c2a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref @@ -5,7 +7,7 @@ module Gitlab def initialize(current_user, project, query, repository_ref = nil, per_page: 20) @current_user = current_user @project = project - @repository_ref = repository_ref.presence || project.default_branch + @repository_ref = repository_ref.presence @query = query @per_page = per_page end @@ -15,9 +17,9 @@ module Gitlab when 'notes' notes.page(page).per(per_page) when 'blobs' - Kaminari.paginate_array(blobs).page(page).per(per_page) + paginated_blobs(blobs, page) when 'wiki_blobs' - Kaminari.paginate_array(wiki_blobs).page(page).per(per_page) + paginated_blobs(wiki_blobs, page) when 'commits' Kaminari.paginate_array(commits).page(page).per(per_page) else @@ -29,6 +31,7 @@ module Gitlab @blobs_count ||= blobs.count end + # rubocop: disable CodeReuse/ActiveRecord def limited_notes_count return @limited_notes_count if defined?(@limited_notes_count) @@ -42,6 +45,7 @@ module Gitlab @limited_notes_count end + # rubocop: enable CodeReuse/ActiveRecord def wiki_blobs_count @wiki_blobs_count ||= wiki_blobs.count @@ -51,36 +55,6 @@ module Gitlab @commits_count ||= commits.count end - def self.parse_search_result(result, project = nil) - ref = nil - filename = nil - basename = nil - data = "" - startline = 0 - - result.each_line.each_with_index do |line, index| - prefix ||= line.match(/^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/)&.tap do |matches| - ref = matches[:ref] - filename = matches[:filename] - startline = matches[:startline] - startline = startline.to_i - index - extname = Regexp.escape(File.extname(filename)) - basename = filename.sub(/#{extname}$/, '') - end - - data << line.sub(prefix.to_s, '') - end - - FoundBlob.new( - filename: filename, - basename: basename, - ref: ref, - startline: startline, - data: data, - project_id: project ? project.id : nil - ) - end - def single_commit_result? return false if commits_count != 1 @@ -92,10 +66,18 @@ module Gitlab private + def paginated_blobs(blobs, page) + results = Kaminari.paginate_array(blobs).page(page).per(per_page) + + Gitlab::Search::FoundBlob.preload_blobs(results) + + results + end + def blobs return [] unless Ability.allowed?(@current_user, :download_code, @project) - @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) + @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query) end def wiki_blobs @@ -103,10 +85,8 @@ module Gitlab @wiki_blobs ||= begin if project.wiki_enabled? && query.present? - project_wiki = ProjectWiki.new(project) - - unless project_wiki.empty? - project_wiki.search_files(query) + unless project.wiki.empty? + Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query) else [] end @@ -120,9 +100,11 @@ module Gitlab @notes ||= notes_finder(nil) end + # rubocop: disable CodeReuse/ActiveRecord def notes_finder(type) NotesFinder.new(project, @current_user, search: query, target_type: type).execute.user.order('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def commits @commits ||= find_commits(query) @@ -149,5 +131,13 @@ module Gitlab def project_ids_relation project end + + def repository_project_ref + @repository_project_ref ||= repository_ref || project.default_branch + end + + def repository_wiki_ref + @repository_wiki_ref ||= repository_ref || project.wiki.default_branch + end end end diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/project_service_logger.rb new file mode 100644 index 00000000000..9b0357d3161 --- /dev/null +++ b/lib/gitlab/project_service_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class ProjectServiceLogger < Gitlab::JsonLogger + def self.file_name_noext + 'integrations_json' + end + end +end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 08f6a54776f..3bfd6ee892c 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ProjectTemplate attr_reader :title, :name, :description, :preview diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb index 690c38737c0..d8f1d1e2316 100644 --- a/lib/gitlab/project_transfer.rb +++ b/lib/gitlab/project_transfer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # This class is used to move local, unhashed files owned by projects to their new location class ProjectTransfer diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index bb1172f82a1..bd4ca578840 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module AdditionalMetricsParser @@ -5,7 +7,7 @@ module Gitlab MUTEX = Mutex.new extend self - def load_groups_from_yaml(file_name = 'additional_metrics.yml') + def load_groups_from_yaml(file_name) yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb index f54b2c6aaff..7ebfc2e25a9 100644 --- a/lib/gitlab/prometheus/metric.rb +++ b/lib/gitlab/prometheus/metric.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Prometheus class Metric include ActiveModel::Model - attr_accessor :title, :required_metrics, :weight, :y_label, :queries + attr_accessor :id, :title, :required_metrics, :weight, :y_label, :queries validates :title, :required_metrics, :weight, :y_label, :queries, presence: true diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index e91c6fb2e27..394556e8708 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -1,13 +1,24 @@ +# frozen_string_literal: true + module Gitlab module Prometheus class MetricGroup include ActiveModel::Model attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true def self.common_metrics - AdditionalMetricsParser.load_groups_from_yaml + all_groups = ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new( + name: name, + priority: metrics.map(&:priority).max, + metrics: metrics.map(&:to_query_metric) + ) + end + + all_groups.sort_by(&:priority).reverse end # EE only diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb index 49cc0e16080..20b5ef5ce55 100644 --- a/lib/gitlab/prometheus/parsing_error.rb +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Prometheus ParsingError = Class.new(StandardError) diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index e677ec84cd4..ab6ef7d5466 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -1,13 +1,17 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( deployment.project, + deployment.environment, common_query_context( deployment.environment, timeframe_start: (deployment.created_at - 30.minutes).to_f, @@ -16,6 +20,7 @@ module Gitlab ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index 9273e69e158..34b705138ba 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -1,17 +1,22 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries class AdditionalMetricsEnvironmentQuery < BaseQuery include QueryAdditionalMetrics + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| query_metrics( environment.project, + environment, common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) ) end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 29cab6e9c15..9ff414d5236 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index c2626581897..fc32c4353f0 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +28,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index b62910c8de6..56195f85a70 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,7 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries class EnvironmentQuery < BaseQuery + # rubocop: disable CodeReuse/ActiveRecord def query(environment_id) ::Environment.find_by(id: environment_id).try do |environment| environment_slug = environment.slug @@ -19,6 +22,7 @@ module Gitlab } end end + # rubocop: enable CodeReuse/ActiveRecord def self.transform_reactive_result(result) result[:metrics] = result.delete :data diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index d920e9a749f..32294756aa2 100644 --- a/lib/gitlab/prometheus/queries/matched_metric_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index f5879de1e94..960d3536ec0 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module Gitlab module Prometheus module Queries module QueryAdditionalMetrics - def query_metrics(project, query_context) + def query_metrics(project, environment, query_context) matched_metrics(project).map(&query_group(query_context)) .select(&method(:group_with_any_metrics)) end @@ -14,12 +16,16 @@ module Gitlab lambda do |group| metrics = group.metrics.map do |metric| - { + metric_hsh = { title: metric.title, weight: metric.weight, y_label: metric.y_label, queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) } + + metric_hsh[:id] = metric.id if metric.id + + metric_hsh end { @@ -77,11 +83,8 @@ module Gitlab end def common_query_context(environment, timeframe_start:, timeframe_end:) - base_query_context(timeframe_start, timeframe_end).merge({ - ci_environment_slug: environment.slug, - kube_namespace: environment.deployment_platform&.actual_namespace || '', - environment_filter: %{container_name!="POD",environment="#{environment.slug}"} - }) + base_query_context(timeframe_start, timeframe_end) + .merge(QueryVariables.call(environment)) end def base_query_context(timeframe_start, timeframe_end) diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb new file mode 100644 index 00000000000..1cc85d4b4a6 --- /dev/null +++ b/lib/gitlab/prometheus/query_variables.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Prometheus + module QueryVariables + def self.call(environment) + { + ci_environment_slug: environment.slug, + kube_namespace: environment.deployment_platform&.actual_namespace || '', + environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + } + end + end + end +end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index b66253a10e0..45828c77a33 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 2819c7d062c..efeb1e07d49 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module ProtocolAccess def self.allowed?(protocol) diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index d682289b632..a64cb47e77e 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + # This class is part of the Gitlab::HTTP wrapper. Depending on the value # of the global setting allow_local_requests_from_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # -# This functionality can be overriden by providing the setting the option +# This functionality can be overridden by providing the setting the option # allow_local_requests = true in the request. For example: # Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true) # diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index 9f69a9e4a39..31e6b120e45 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QueryLimiting # Returns true if we should enable tracking of query counts. diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 3c4ff5d1928..065862174bb 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QueryLimiting class ActiveSupportSubscriber < ActiveSupport::Subscriber diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 66d7d9275cf..e8fad067fa6 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QueryLimiting class Transaction @@ -68,7 +70,7 @@ module Gitlab def error_message header = 'Too many SQL queries were executed' - header += " in #{action}" if action + header = "#{header} in #{action}" if action "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" end diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 96415271316..259345b8a9a 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -1,14 +1,17 @@ +# frozen_string_literal: true + module Gitlab module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block + :condition_block, :parse_params_block, :action_block, :warning def initialize(name, attributes = {}) @name = name @aliases = attributes[:aliases] || [] @description = attributes[:description] || '' + @warning = attributes[:warning] || '' @explanation = attributes[:explanation] || '' @params = attributes[:params] || [] @condition_block = attributes[:condition_block] @@ -33,11 +36,13 @@ module Gitlab def explain(context, arg) return unless available?(context) - if explanation.respond_to?(:call) - execute_block(explanation, context, arg) - else - explanation - end + message = if explanation.respond_to?(:call) + execute_block(explanation, context, arg) + else + explanation + end + + warning.empty? ? message : "#{message} (#{warning})" end def execute(context, arg) @@ -61,6 +66,7 @@ module Gitlab name: name, aliases: aliases, description: desc, + warning: warning, params: prms } end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index d82dccd0db5..a3aab92061b 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QuickActions module Dsl @@ -31,6 +33,10 @@ module Gitlab @description = block_given? ? block : text end + def warning(message = '') + @warning = message + end + # Allows to define params for the next quick action. # These params are shown in the autocomplete menu. # @@ -133,6 +139,7 @@ module Gitlab name, aliases: aliases, description: @description, + warning: @warning, explanation: @explanation, params: @params, condition_block: @condition_block, @@ -150,6 +157,7 @@ module Gitlab @explanation = nil @params = nil @condition_block = nil + @warning = nil @parse_params_block = nil end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 075ff91700c..ff9bb293b47 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QuickActions # This class takes an array of commands that should be extracted from a @@ -29,7 +31,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content) + def extract_commands(content, only: nil) return [content, []] unless content content = content.dup @@ -37,9 +39,9 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex) do + content.gsub!(commands_regex(only: only)) do if $~[:cmd] - commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' else $~[0] @@ -60,8 +62,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex - names = command_names.map(&:to_s) + def commands_regex(only:) + names = command_names(limit_to_commands: only).map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -102,14 +104,14 @@ module Gitlab # /close ^\/ - (?<cmd>#{Regexp.union(names)}) + (?<cmd>#{Regexp.new(Regexp.union(names).source, Regexp::IGNORECASE)}) (?: [ ] (?<arg>[^\n]*) )? (?:\n|$) ) - }mx + }mix end def perform_substitutions(content, commands) @@ -120,7 +122,7 @@ module Gitlab end substitution_definitions.each do |substitution| - match_data = substitution.match(content) + match_data = substitution.match(content.downcase) if match_data command = [substitution.name.to_s] command << match_data[1] unless match_data[1].empty? @@ -133,10 +135,14 @@ module Gitlab [content, commands] end - def command_names + def command_names(limit_to_commands:) command_definitions.flat_map do |command| next if command.noop? + if limit_to_commands && (command.all_names & limit_to_commands).empty? + next + end + command.all_names end.compact end diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 7328c517a30..f5176376a60 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QuickActions # This class takes spend command argument diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 032c49ed159..2f78ea05cf0 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module QuickActions class SubstitutionDefinition < CommandDefinition @@ -15,7 +17,7 @@ module Gitlab return unless content all_names.each do |a_name| - content.gsub!(%r{/#{a_name} ?(.*)$}, execute_block(action_block, context, '\1')) + content.gsub!(%r{/#{a_name} ?(.*)$}i, execute_block(action_block, context, '\1')) end content end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index c9efa28d7e7..6559c3e3c57 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Recaptcha def self.load_configurations! diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index a991933e910..6e8403ad878 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # please require all dependencies below: require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index e1695aafbeb..8b42c269dd0 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # please require all dependencies below: require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index e5a0fdae7ef..9066606ca21 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # please require all dependencies below: require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 4178b436acf..07a1e20b076 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This file should only be used by sub-classes, not directly by any clients of the sub-classes # please require all dependencies below: require 'active_support/core_ext/hash/keys' diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index bb26f1b610a..d2dbc6f5ef5 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class ReferenceCounter REFERENCE_EXPIRE_TIME = 600 diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 9ff82d628c0..00f817c2399 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ac3de2a8f71..7a1a2eaf6c0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Regex extend self @@ -73,5 +75,35 @@ module Gitlab def build_trace_section_regex @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze end + + def markdown_code_or_html_blocks + @markdown_code_or_html_blocks ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `>>>` blocks which are ignored by this filter + # ``` + + ^``` + .+? + \n```\ *$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + + ^<[^>]+?>\ *\n + .+? + \n<\/[^>]+?>\ *$ + ) + }mx + end + + def jira_transition_id_regex + @jira_transition_id_regex ||= /\d+/ + end end end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 4888184403c..202d310e237 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module RepoPath NotFoundError = Class.new(StandardError) diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index b1bf3ca4143..56007574b1b 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Interface to the Redis-backed cache store module Gitlab class RepositoryCache @@ -6,7 +8,7 @@ module Gitlab def initialize(repository, extra_namespace: nil, backend: Rails.cache) @repository = repository @namespace = "#{repository.full_path}:#{repository.project.id}" - @namespace += ":#{extra_namespace}" if extra_namespace + @namespace = "#{@namespace}:#{extra_namespace}" if extra_namespace @backend = backend end @@ -29,5 +31,21 @@ module Gitlab def read(key) backend.read(cache_key(key)) end + + def write(key, value) + backend.write(cache_key(key), value) + end + + def fetch_without_caching_false(key, &block) + value = read(key) + return value if value + + value = yield + + # Don't cache false values + write(key, value) if value + + value + end end end diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 7f64a8c9e46..931298b5117 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -1,23 +1,82 @@ +# frozen_string_literal: true + module Gitlab module RepositoryCacheAdapter extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize class_methods do - # Wraps around the given method and caches its output in Redis and an instance - # variable. + # Caches and strongly memoizes the method. # # This only works for methods that do not take any arguments. - def cache_method(name, fallback: nil, memoize_only: false) - original = :"_uncached_#{name}" + # + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) - alias_method(original, name) + define_method(name) do + cache_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Caches truthy values from the method. All values are strongly memoized, + # and cached in RequestStore. + # + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be cached. + def cache_method_asymmetrically(name) + uncached_name = alias_uncached_method(name) define_method(name) do - cache_method_output(name, fallback: fallback, memoize_only: memoize_only) do - __send__(original) # rubocop:disable GitlabSecurity/PublicSend + cache_method_output_asymmetrically(name) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend end end end + + # Strongly memoizes the method. + # + # This only works for methods that do not take any arguments. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value + # is not memoized. + def memoize_method(name, fallback: nil) + uncached_name = alias_uncached_method(name) + + define_method(name) do + memoize_method_output(name, fallback: fallback) do + __send__(uncached_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # Prepends "_uncached_" to the target method name + # + # Returns the uncached method name + def alias_uncached_method(name) + uncached_name = :"_uncached_#{name}" + + alias_method(uncached_name, name) + + uncached_name + end + end + + # RequestStore-backed RepositoryCache to be used. Should be overridden by + # the including class + def request_store_cache + raise NotImplementedError end # RepositoryCache to be used. Should be overridden by the including class @@ -25,60 +84,98 @@ module Gitlab raise NotImplementedError end - # Caches the supplied block both in a cache and in an instance variable. + # List of cached methods. Should be overridden by the including class + def cached_methods + raise NotImplementedError + end + + # Caches and strongly memoizes the supplied block. # - # The cache key and instance variable are named the same way as the value of - # the `key` argument. + # name - The name of the method to be cached. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. + def cache_method_output(name, fallback: nil, &block) + memoize_method_output(name, fallback: fallback) do + cache.fetch(name, &block) + end + end + + # Caches truthy values from the supplied block. All values are strongly + # memoized, and cached in RequestStore. # - # This method will return `nil` if the corresponding instance variable is also - # set to `nil`. This ensures we don't keep yielding the block when it returns - # `nil`. + # Currently only used to cache `exists?` since stale false values are + # particularly troublesome. This can occur, for example, when an NFS mount + # is temporarily down. # - # key - The name of the key to cache the data in. - # fallback - A value to fall back to in the event of a Git error. - def cache_method_output(key, fallback: nil, memoize_only: false, &block) - ivar = cache_instance_variable_name(key) - - if instance_variable_defined?(ivar) - instance_variable_get(ivar) - else - # If the repository doesn't exist and a fallback was specified we return - # that value inmediately. This saves us Rugged/gRPC invocations. - return fallback unless fallback.nil? || cache.repository.exists? - - begin - value = - if memoize_only - yield - else - cache.fetch(key, &block) - end - - instance_variable_set(ivar, value) - rescue Gitlab::Git::Repository::NoRepository - # Even if the above `#exists?` check passes these errors might still - # occur (for example because of a non-existing HEAD). We want to - # gracefully handle this and not cache anything - fallback + # name - The name of the method to be cached. + def cache_method_output_asymmetrically(name, &block) + memoize_method_output(name) do + request_store_cache.fetch(name) do + cache.fetch_without_caching_false(name, &block) end end end + # Strongly memoizes the supplied block. + # + # name - The name of the method to be memoized. + # fallback - A value to fall back to if the repository does not exist, or + # in case of a Git error. Defaults to nil. The fallback value is + # not memoized. + def memoize_method_output(name, fallback: nil, &block) + no_repository_fallback(name, fallback: fallback) do + strong_memoize(memoizable_name(name), &block) + end + end + + # Returns the fallback value if the repository does not exist + def no_repository_fallback(name, fallback: nil, &block) + # Avoid unnecessary gRPC invocations + return fallback if fallback && fallback_early?(name) + + yield + rescue Gitlab::Git::Repository::NoRepository + # Even if the `#exists?` check in `fallback_early?` passes, these errors + # might still occur (for example because of a non-existing HEAD). We + # want to gracefully handle this and not memoize anything. + fallback + end + # Expires the caches of a specific set of methods def expire_method_caches(methods) - methods.each do |key| - cache.expire(key) + methods.each do |name| + unless cached_methods.include?(name.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{name}' for Repository" + next + end - ivar = cache_instance_variable_name(key) + cache.expire(name) - remove_instance_variable(ivar) if instance_variable_defined?(ivar) + clear_memoization(memoizable_name(name)) end + + expire_request_store_method_caches(methods) end private - def cache_instance_variable_name(key) - :"@#{key.to_s.tr('?!', '')}" + def memoizable_name(name) + "#{name.to_s.tr('?!', '')}" + end + + def expire_request_store_method_caches(methods) + methods.each do |name| + request_store_cache.expire(name) + end + end + + # All cached repository methods depend on the existence of a Git repository, + # so if the repository doesn't exist, we already know not to call it. + def fallback_early?(method_name) + # Avoid infinite loop + return false if method_name == :exists? + + !exists? end end end diff --git a/lib/gitlab/repository_check_logger.rb b/lib/gitlab/repository_check_logger.rb index 485b596ca57..e90b0a002af 100644 --- a/lib/gitlab/repository_check_logger.rb +++ b/lib/gitlab/repository_check_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class RepositoryCheckLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index fef536ecb0b..d9811e036d3 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module Gitlab class RequestContext class << self def client_ip - RequestStore[:client_ip] + Gitlab::SafeRequestStore[:client_ip] end end @@ -11,9 +13,9 @@ module Gitlab end def call(env) - req = Rack::Request.new(env) + req = ActionDispatch::Request.new(env) - RequestStore[:client_ip] = req.ip + Gitlab::SafeRequestStore[:client_ip] = req.ip @app.call(env) end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index ccfe0d6bed3..b1e478093d3 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # A module to check CSRF tokens in requests. # It's used in API helpers and OmniAuth. # Usage: GitLab::RequestForgeryProtection.call(env) @@ -5,7 +7,7 @@ module Gitlab module RequestForgeryProtection class Controller < ActionController::Base - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true rescue_from ActionController::InvalidAuthenticityToken do |e| logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 0c9ab759e81..64593153686 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'fileutils' module Gitlab diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index ef42b0557e0..7615f6f443b 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'ruby-prof' module Gitlab diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index f89d56903ef..46996ef8c51 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module RequestProfiler class Profile diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index f3952657983..a555bf1d812 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class RouteMap FormatError = Class.new(StandardError) diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 2c994536060..3b05f181ed2 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Routing extend ActiveSupport::Concern @@ -47,7 +49,7 @@ module Gitlab # # `request.fullpath` includes the querystring new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") - new_path << "?#{request.query_string}" if request.query_string.present? + new_path = "#{new_path}?#{request.query_string}" if request.query_string.present? new_path end diff --git a/lib/gitlab/safe_request_store.rb b/lib/gitlab/safe_request_store.rb new file mode 100644 index 00000000000..d146913bdb3 --- /dev/null +++ b/lib/gitlab/safe_request_store.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module SafeRequestStore + NULL_STORE = Gitlab::NullRequestStore.new + + class << self + # These methods should always run directly against RequestStore + delegate :clear!, :begin!, :end!, :active?, to: :RequestStore + + # These methods will run against NullRequestStore if RequestStore is disabled + delegate :read, :[], :write, :[]=, :exist?, :fetch, :delete, to: :store + end + + def self.store + if RequestStore.active? + RequestStore + else + NULL_STORE + end + end + + # This method accept an options hash to be compatible with + # ActiveSupport::Cache::Store#write method. The options are + # not passed to the underlying cache implementation because + # RequestStore#write accepts only a key, and value params. + def self.write(key, value, options = nil) + store.write(key, value) + end + end +end diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index 8304b9a482c..0d4e6be2129 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sanitizers module SVG diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb index d50f826f924..b4da24b3215 100644 --- a/lib/gitlab/sanitizers/svg/whitelist.rb +++ b/lib/gitlab/sanitizers/svg/whitelist.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Generated from: # SVG element list: https://www.w3.org/TR/SVG/eltindex.html # SVG Attribute list: https://www.w3.org/TR/SVG/attindex.html diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb new file mode 100644 index 00000000000..a62ab1521a7 --- /dev/null +++ b/lib/gitlab/search/found_blob.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class FoundBlob + include EncodingHelper + include Presentable + include BlobLanguageFromGitAttributes + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :content_match, :blob_filename + + FILENAME_REGEXP = /\A(?<ref>[^:]*):(?<filename>[^\x00]*)\x00/.freeze + CONTENT_REGEXP = /^(?<ref>[^:]*):(?<filename>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze + + def self.preload_blobs(blobs) + to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_filename } + + to_fetch.each { |blob| blob.fetch_blob } + end + + def initialize(opts = {}) + @id = opts.fetch(:id, nil) + @binary_filename = opts.fetch(:filename, nil) + @binary_basename = opts.fetch(:basename, nil) + @ref = opts.fetch(:ref, nil) + @startline = opts.fetch(:startline, nil) + @binary_data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) + @project = opts.fetch(:project, nil) + # Some caller does not have project object (e.g. elastic search), + # yet they can trigger many calls in one go, + # causing duplicated queries. + # Allow those to just pass project_id instead. + @project_id = opts.fetch(:project_id, nil) + @content_match = opts.fetch(:content_match, nil) + @blob_filename = opts.fetch(:blob_filename, nil) + @repository = opts.fetch(:repository, nil) + end + + def id + @id ||= parsed_content[:id] + end + + def ref + @ref ||= parsed_content[:ref] + end + + def startline + @startline ||= parsed_content[:startline] + end + + # binary_filename is used for running filters on all matches, + # for grepped results (which use content_match), we get + # filename from the beginning of the grepped result which is faster + # then parsing whole snippet + def binary_filename + @binary_filename ||= content_match ? search_result_filename : parsed_content[:binary_filename] + end + + def filename + @filename ||= encode_utf8(@binary_filename || parsed_content[:binary_filename]) + end + + def basename + @basename ||= encode_utf8(@binary_basename || parsed_content[:binary_basename]) + end + + def data + @data ||= encode_utf8(@binary_data || parsed_content[:binary_data]) + end + + def path + filename + end + + def project_id + @project_id || @project&.id + end + + def present + super(presenter_class: BlobPresenter) + end + + def fetch_blob + path = [ref, blob_filename] + missing_blob = { binary_filename: blob_filename } + + BatchLoader.for(path).batch(default_value: missing_blob) do |refs, loader| + Gitlab::Git::Blob.batch(repository, refs, blob_size_limit: 1024).each do |blob| + # if the blob couldn't be fetched for some reason, + # show at least the blob filename + data = { + id: blob.id, + binary_filename: blob.path, + binary_basename: File.basename(blob.path, File.extname(blob.path)), + ref: ref, + startline: 1, + binary_data: blob.data, + project: project + } + + loader.call([ref, blob.path], data) + end + end + end + + private + + def search_result_filename + content_match.match(FILENAME_REGEXP) { |matches| matches[:filename] } + end + + def parsed_content + strong_memoize(:parsed_content) do + if content_match + parse_search_result + elsif blob_filename + fetch_blob + else + {} + end + end + end + + def parse_search_result + ref = nil + filename = nil + basename = nil + + data = [] + startline = 0 + + content_match.each_line.each_with_index do |line, index| + prefix ||= line.match(CONTENT_REGEXP)&.tap do |matches| + ref = matches[:ref] + filename = matches[:filename] + startline = matches[:startline] + startline = startline.to_i - index + extname = Regexp.escape(File.extname(filename)) + basename = filename.sub(/#{extname}$/, '') + end + + data << line.sub(prefix.to_s, '') + end + + { + binary_filename: filename, + binary_basename: basename, + ref: ref, + startline: startline, + binary_data: data.join, + project: project + } + end + + def repository + @repository ||= project.repository + end + end + end +end diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb new file mode 100644 index 00000000000..c4fb0199558 --- /dev/null +++ b/lib/gitlab/search/parsed_query.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class ParsedQuery + attr_reader :term, :filters + + def initialize(term, filters) + @term = term + @filters = filters + end + + def filter_results(results) + filters = @filters.reject { |filter| filter[:matcher].nil? } + return unless filters + + results.select do |result| + filters.all? do |filter| + filter[:matcher].call(filter, result) + end + end + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb new file mode 100644 index 00000000000..ba0e16607a6 --- /dev/null +++ b/lib/gitlab/search/query.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class Query < SimpleDelegator + include EncodingHelper + + def initialize(query, filter_opts = {}, &block) + @raw_query = query.dup + @filters = [] + @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts) + + self.instance_eval(&block) if block_given? + + @query = Gitlab::Search::ParsedQuery.new(*extract_filters) + # set the ParsedQuery as our default delegator thanks to SimpleDelegator + super(@query) + end + + private + + def filter(name, **attributes) + filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + + @filters << filter + end + + def filter_options(**options) + @filter_options.merge!(options) + end + + def extract_filters + fragments = [] + + filters = @filters.each_with_object([]) do |filter, parsed_filters| + match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + next unless match + + input = match.split(':')[1..-1].join + next if input.empty? + + filter[:value] = parse_filter(filter, input) + filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') + fragments << match + + parsed_filters << filter + end + + query = (@raw_query.split - fragments).join(' ') + + [query, filters] + end + + def parse_filter(filter, input) + result = filter[:parser].call(input) + + @filter_options[:encode_binary] ? encode_binary(result) : result + end + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 1e45d074e0a..491148ec1a6 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -1,30 +1,7 @@ +# frozen_string_literal: true + module Gitlab class SearchResults - class FoundBlob - include EncodingHelper - - attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id - - def initialize(opts = {}) - @id = opts.fetch(:id, nil) - @filename = encode_utf8(opts.fetch(:filename, nil)) - @basename = encode_utf8(opts.fetch(:basename, nil)) - @ref = opts.fetch(:ref, nil) - @startline = opts.fetch(:startline, nil) - @data = encode_utf8(opts.fetch(:data, nil)) - @per_page = opts.fetch(:per_page, 20) - @project_id = opts.fetch(:project_id, nil) - end - - def path - filename - end - - def no_highlighting? - false - end - end - attr_reader :current_user, :query, :per_page # Limit search results by passed projects @@ -62,10 +39,13 @@ module Gitlab without_count ? collection.without_count : collection end + # rubocop: disable CodeReuse/ActiveRecord def limited_projects_count @limited_projects_count ||= projects.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_issues_count return @limited_issues_count if @limited_issues_count @@ -77,14 +57,19 @@ module Gitlab sum = issues(public_only: true).limit(count_limit).count @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_merge_requests_count @limited_merge_requests_count ||= merge_requests.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def limited_milestones_count @limited_milestones_count ||= milestones.limit(count_limit).count end + # rubocop: enable CodeReuse/ActiveRecord def single_commit_result? false @@ -100,6 +85,7 @@ module Gitlab limit_projects.search(query) end + # rubocop: disable CodeReuse/ActiveRecord def issues(finder_params = {}) issues = IssuesFinder.new(current_user, finder_params).execute unless default_project_filter @@ -115,13 +101,17 @@ module Gitlab issues.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) milestones.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def merge_requests merge_requests = MergeRequestsFinder.new(current_user).execute unless default_project_filter @@ -137,13 +127,16 @@ module Gitlab merge_requests.reorder('updated_at DESC') end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'projects' end + # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation limit_projects.select(:id).reorder(nil) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 98f005cb61b..8e2f16271eb 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # :nocov: module DeliverNever def deliver_later @@ -24,6 +26,19 @@ module Gitlab puts "\nOK".color(:green) end + def self.without_gitaly_timeout + # Remove Gitaly timeout + old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0) + # Otherwise we still see the default value when running seed_fu + ApplicationSetting.expire + + yield + ensure + Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout) + ApplicationSetting.expire + end + def self.mute_notifications NotificationService.prepend(MuteNotifications) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 6381e94c1d2..956c16117f5 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,11 +1,14 @@ +# frozen_string_literal: true + module Gitlab module Sentry def self.enabled? - Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? + (Rails.env.production? || Rails.env.development?) && + Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) - return unless self.enabled? + return unless enabled? Raven.tags_context(locale: I18n.locale) @@ -27,25 +30,29 @@ module Gitlab # # Provide an issue URL for follow up. def self.track_exception(exception, issue_url: nil, extra: {}) + track_acceptable_exception(exception, issue_url: issue_url, extra: extra) + + raise exception if should_raise_for_dev? + end + + # This should be used when you do not want to raise an exception in + # development and test. If you need development and test to behave + # just the same as production you can use this instead of + # track_exception. + def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? extra[:issue_url] = issue_url if issue_url context # Make sure we've set everything we know in the context - Raven.capture_exception(exception, extra: extra) - end - - raise exception if should_raise? - end + tags = { + Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id + } - def self.program_context - if Sidekiq.server? - 'sidekiq' - else - 'rails' + Raven.capture_exception(exception, tags: tags, extra: extra) end end - def self.should_raise? + def self.should_raise_for_dev? Rails.env.development? || Rails.env.test? end end diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb index c059c454eac..9abf3a54f37 100644 --- a/lib/gitlab/serializer/ci/variables.rb +++ b/lib/gitlab/serializer/ci/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Serializer module Ci @@ -13,8 +15,9 @@ module Gitlab object = YAML.safe_load(string, [Symbol]) object.map do |variable| - variable[:key] = variable[:key].to_s - variable + variable.symbolize_keys.tap do |variable| + variable[:key] = variable[:key].to_s + end end end diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb index 6bb00d8ae21..eb242cc7c20 100644 --- a/lib/gitlab/serializer/pagination.rb +++ b/lib/gitlab/serializer/pagination.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Serializer class Pagination diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index e5c02dd8ecc..2b7e12639be 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true + +require 'toml-rb' + module Gitlab module SetupHelper class << self @@ -9,7 +13,7 @@ module Gitlab # because it uses a Unix socket. # For development and testing purposes, an extra storage is added to gitaly, # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + def gitaly_configuration_toml(gitaly_dir, storage_paths, gitaly_ruby: true) storages = [] address = nil @@ -24,11 +28,14 @@ module Gitlab address = val['gitaly_address'] end - storages << { name: key, path: val.legacy_disk_path } + storages << { name: key, path: storage_paths[key] } end if Rails.env.test? - storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s + + FileUtils.mkdir(storage_path) unless File.exist?(storage_path) + storages << { name: 'test_second_storage', path: storage_path } end config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } @@ -41,12 +48,12 @@ module Gitlab end # rubocop:disable Rails/Output - def create_gitaly_configuration(dir, force: false) + def create_gitaly_configuration(dir, storage_paths, force: false) config_path = File.join(dir, 'config.toml') FileUtils.rm_f(config_path) if force File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml(dir) + f.puts gitaly_configuration_toml(dir, storage_paths) end rescue Errno::EEXIST puts "Skipping config.toml generation:" diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb new file mode 100644 index 00000000000..6612347438e --- /dev/null +++ b/lib/gitlab/shard_health_cache.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + class ShardHealthCache + HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze + HEALTHY_SHARDS_TIMEOUT = 300 + + # Clears the Redis set storing the list of healthy shards + def self.clear + Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + end + + # Updates the list of healthy shards using a Redis set + # + # shards - An array of shard names to store + def self.update(shards) + Gitlab::Redis::Cache.with do |redis| + redis.multi do |m| + m.del(HEALTHY_SHARDS_KEY) + shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) + end + end + end + + # Returns an array of strings of healthy shards + def self.cached_healthy_shards + Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + end + + # Checks whether the given shard name is in the list of healthy shards. + # + # shard_name - The string to check + def self.healthy_shard?(shard_name) + Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + end + + # Returns the number of healthy shards in the Redis set + def self.healthy_shard_count + Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4a691d640b3..bdf21cf3134 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,5 +1,6 @@ -# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. -# SSH key operations are not part of Gitaly so will never be migrated. +# frozen_string_literal: true + +# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated. require 'securerandom' @@ -75,17 +76,10 @@ module Gitlab relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') - gitaly_migrate(:create_repository, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - repository = Gitlab::Git::Repository.new(storage, relative_path, '') - repository.gitaly_repository_client.create_repository - true - else - repo_path = File.join(Gitlab.config.repositories.storages[storage].legacy_disk_path, relative_path) - Gitlab::Git::Repository.create(repo_path, bare: true, symlink_hooks_to: gitlab_shell_hooks_path) - end - end + repository = Gitlab::Git::Repository.new(storage, relative_path, '') + wrapped_gitaly_errors { repository.gitaly_repository_client.create_repository } + + true rescue => err # Once the Rugged codes gets removes this can be improved Rails.logger.error("Failed to add repository #{storage}/#{name}: #{err}") false @@ -100,42 +94,20 @@ module Gitlab # Ex. # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) if url.start_with?('.', '/') raise Error.new("don't use disk paths with import_repository: #{url.inspect}") end - # The timeout ensures the subprocess won't hang forever - cmd = gitlab_projects(storage, "#{name}.git") - success = cmd.import_project(url, git_timeout) + relative_path = "#{name}.git" + cmd = GitalyGitlabProjects.new(storage, relative_path) + success = cmd.import_project(url, git_timeout) raise Error, cmd.output unless success success end - # Fetch remote for repository - # - # repository - an instance of Git::Repository - # remote - remote name - # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication - # forced - should we use --force flag? - # no_tags - should we use --no-tags flag? - # - # Ex. - # fetch_remote(my_repo, "upstream") - # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - gitaly_migrate(:fetch_remote) do |is_enabled| - if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - else - local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) - end - end - end - # Move repository reroutes to mv_directory which is an alias for # mv_namespace. Given the underlying implementation is a move action, # indescriminate of what the folders might be. @@ -146,8 +118,6 @@ module Gitlab # # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) return false if path.empty? || new_path.empty? @@ -162,11 +132,11 @@ module Gitlab # # Ex. # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) - gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") - .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") + forked_from_relative_path = "#{forked_from_disk_path}.git" + fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] + + GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -178,8 +148,6 @@ module Gitlab # # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) return false if name.empty? @@ -242,6 +210,7 @@ module Gitlab # Ex. # remove_keys_not_found_in_db # + # rubocop: disable CodeReuse/ActiveRecord def remove_keys_not_found_in_db return unless self.authorized_keys_enabled? @@ -260,6 +229,7 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord # Iterate over all ssh key IDs from gitlab shell, in batches # @@ -319,10 +289,12 @@ module Gitlab # def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) - rescue GRPC::InvalidArgument + rescue GRPC::InvalidArgument => e + Gitlab::Sentry.track_acceptable_exception(e, extra: { old_name: old_name, new_name: new_name, storage: storage }) + false end - alias_method :mv_directory, :mv_namespace + alias_method :mv_directory, :mv_namespace # Note: ShellWorker uses this alias def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -343,9 +315,11 @@ module Gitlab # exists?(storage, 'gitlab') # exists?(storage, 'gitlab/cookies.git') # + # rubocop: disable CodeReuse/ActiveRecord def exists?(storage, dir_name) Gitlab::GitalyClient::NamespaceService.new(storage).exists?(dir_name) end + # rubocop: enable CodeReuse/ActiveRecord protected @@ -385,37 +359,6 @@ module Gitlab private - def gitlab_projects(shard_name, disk_path) - Gitlab::Git::GitlabProjects.new( - shard_name, - disk_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - end - - def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - vars = { force: forced, tags: !no_tags, prune: prune } - - if ssh_auth&.ssh_import? - if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? - vars[:ssh_key] = ssh_auth.ssh_private_key - end - - if ssh_auth.ssh_known_hosts.present? - vars[:known_hosts] = ssh_auth.ssh_known_hosts - end - end - - cmd = gitlab_projects(storage_name, repository_relative_path) - - success = cmd.fetch_remote(remote, git_timeout, vars) - - raise Error, cmd.output unless success - - success - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) @@ -445,12 +388,46 @@ module Gitlab Gitlab.config.gitlab_shell.git_timeout end - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) + def wrapped_gitaly_errors + yield rescue GRPC::NotFound, GRPC::BadStatus => e # Old Popen code returns [Error, output] to the caller, so we # need to do the same here... raise Error, e end + + class GitalyGitlabProjects + attr_reader :shard_name, :repository_relative_path, :output + + def initialize(shard_name, repository_relative_path) + @shard_name = shard_name + @repository_relative_path = repository_relative_path + @output = '' + end + + def import_project(source, _timeout) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source) + true + rescue GRPC::BadStatus => e + @output = e.message + false + end + + def fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end + + def logger + Rails.logger + end + end end end diff --git a/lib/gitlab/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index 053dd4ab9e0..59fc6ee8dc8 100644 --- a/lib/gitlab/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # == GitLab Shell mixin # # Provide a shortcut to Gitlab::Shell instance by gitlab_shell diff --git a/lib/gitlab/sherlock.rb b/lib/gitlab/sherlock.rb index 6360527a7aa..a1471c9de47 100644 --- a/lib/gitlab/sherlock.rb +++ b/lib/gitlab/sherlock.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'securerandom' module Gitlab diff --git a/lib/gitlab/sherlock/collection.rb b/lib/gitlab/sherlock/collection.rb index 66bd6258521..ce3a376cf75 100644 --- a/lib/gitlab/sherlock/collection.rb +++ b/lib/gitlab/sherlock/collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock # A collection of transactions recorded by Sherlock. diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 89072b01f2e..604b6df12cc 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock class FileSample diff --git a/lib/gitlab/sherlock/line_profiler.rb b/lib/gitlab/sherlock/line_profiler.rb index b5f9d040047..209ba784f9c 100644 --- a/lib/gitlab/sherlock/line_profiler.rb +++ b/lib/gitlab/sherlock/line_profiler.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock # Class for profiling code on a per line basis. diff --git a/lib/gitlab/sherlock/line_sample.rb b/lib/gitlab/sherlock/line_sample.rb index eb1948eb6d6..c92fa9ea1ff 100644 --- a/lib/gitlab/sherlock/line_sample.rb +++ b/lib/gitlab/sherlock/line_sample.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock class LineSample diff --git a/lib/gitlab/sherlock/location.rb b/lib/gitlab/sherlock/location.rb index 5ac265618ad..4bba60f3490 100644 --- a/lib/gitlab/sherlock/location.rb +++ b/lib/gitlab/sherlock/location.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock class Location diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 4c88e33699a..747cb0f9142 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock # Rack middleware used for tracking request metrics. diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 02ddc3f47eb..11561eec32a 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock class Query @@ -48,7 +50,7 @@ module Gitlab end unless @query.end_with?(';') - @query += ';' + @query = "#{@query};" end end diff --git a/lib/gitlab/sherlock/transaction.rb b/lib/gitlab/sherlock/transaction.rb index 400a552bf99..d04624977dc 100644 --- a/lib/gitlab/sherlock/transaction.rb +++ b/lib/gitlab/sherlock/transaction.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Sherlock class Transaction diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index c3d7814551c..3b8de64913b 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -1,13 +1,22 @@ +# frozen_string_literal: true + require 'yaml' require 'set' module Gitlab module SidekiqConfig + QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze + # This method is called by `bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. def self.worker_queues(rails_path = Rails.root.to_s) @worker_queues ||= {} - @worker_queues[rails_path] ||= YAML.load_file(File.join(rails_path, 'app/workers/all_queues.yml')) + + @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| + full_path = File.join(rails_path, path) + + File.exist?(full_path) ? YAML.load_file(full_path) : [] + end end # This method is called by `bin/sidekiq-cluster` in EE, which runs outside diff --git a/lib/gitlab/sidekiq_logger.rb b/lib/gitlab/sidekiq_logger.rb index c1dab87a432..ce82a6f04bb 100644 --- a/lib/gitlab/sidekiq_logger.rb +++ b/lib/gitlab/sidekiq_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class SidekiqLogger < Gitlab::Logger def self.file_name_noext diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index 98f8222fd03..88888c5994e 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqLogging class JSONFormatter diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 9a89ae70b98..fdc0d518c59 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -1,8 +1,11 @@ +# frozen_string_literal: true + module Gitlab module SidekiqLogging class StructuredLogger START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) started_at = current_time @@ -62,6 +65,7 @@ module Gitlab job['pid'] = ::Process.pid job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + job['args'] = limited_job_args(job['args']) if job['args'] convert_to_iso8601(job, START_TIMESTAMP_FIELDS) @@ -91,6 +95,21 @@ module Gitlab Time.at(timestamp).utc.iso8601(3) end + + def limited_job_args(args) + return unless args.is_a?(Array) + + total_length = 0 + limited_args = args.take_while do |arg| + total_length += arg.to_json.length + + total_length <= MAXIMUM_JOB_ARGUMENTS_LENGTH + end + + limited_args.push('...') if total_length > MAXIMUM_JOB_ARGUMENTS_LENGTH + + limited_args + end end end end diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 82a59a7a87e..2859aa5f4a6 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqMiddleware class ArgumentsLogger diff --git a/lib/gitlab/sidekiq_middleware/batch_loader.rb b/lib/gitlab/sidekiq_middleware/batch_loader.rb new file mode 100644 index 00000000000..75c4efc3042 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/batch_loader.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class BatchLoader + def call(worker, job, queue) + yield + ensure + ::BatchLoader::Executor.clear_current + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb new file mode 100644 index 00000000000..b807b3a03ed --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class CorrelationInjector + def call(worker_class, job, queue, redis_pool) + job[Gitlab::CorrelationId::LOG_KEY] ||= + Gitlab::CorrelationId.current_or_new_id + + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb new file mode 100644 index 00000000000..cb8ff4a6284 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class CorrelationLogger + def call(worker, job, queue) + correlation_id = job[Gitlab::CorrelationId::LOG_KEY] + + Gitlab::CorrelationId.use_id(correlation_id) do + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb index b1fa0e3cb4e..8824f81e8e3 100644 --- a/lib/gitlab/sidekiq_middleware/request_store_middleware.rb +++ b/lib/gitlab/sidekiq_middleware/request_store_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqMiddleware class RequestStoreMiddleware diff --git a/lib/gitlab/sidekiq_middleware/shutdown.rb b/lib/gitlab/sidekiq_middleware/shutdown.rb index b232ac4da33..19f3be83bce 100644 --- a/lib/gitlab/sidekiq_middleware/shutdown.rb +++ b/lib/gitlab/sidekiq_middleware/shutdown.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'mutex_m' module Gitlab diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index a1f689d94d9..583a970bf4e 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # The SidekiqStatus module and its child classes can be used for checking if a # Sidekiq job has been processed or not. diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb index 00983b3284a..bfd5038557d 100644 --- a/lib/gitlab/sidekiq_status/client_middleware.rb +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqStatus class ClientMiddleware diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb index ceab10b8301..01bc58fd2be 100644 --- a/lib/gitlab/sidekiq_status/server_middleware.rb +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqStatus class ServerMiddleware diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb deleted file mode 100644 index 5512afa45a8..00000000000 --- a/lib/gitlab/sidekiq_throttler.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Gitlab - class SidekiqThrottler - class << self - def execute! - if Gitlab::CurrentSettings.sidekiq_throttling_enabled? - require 'sidekiq-limit_fetch' - - Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| - Sidekiq::Queue[queue].limit = queue_limit - end - end - end - - private - - def queue_limit - @queue_limit ||= - begin - factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor - (factor * Sidekiq.options[:concurrency]).ceil - end - end - end - end -end diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb index 9683214ec18..8164a5a9d7a 100644 --- a/lib/gitlab/sidekiq_versioning.rb +++ b/lib/gitlab/sidekiq_versioning.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqVersioning def self.install! diff --git a/lib/gitlab/sidekiq_versioning/manager.rb b/lib/gitlab/sidekiq_versioning/manager.rb index 308be0fdf76..e5852b43003 100644 --- a/lib/gitlab/sidekiq_versioning/manager.rb +++ b/lib/gitlab/sidekiq_versioning/manager.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SidekiqVersioning module Manager diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index 466554e398c..fcc120112f2 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class BaseCommand @@ -40,9 +42,11 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_by_iid(iid) collection.find_by(iid: iid) end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index bb778f37096..474c09b9c4d 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -1,13 +1,17 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class Command < BaseCommand - COMMANDS = [ - Gitlab::SlashCommands::IssueShow, - Gitlab::SlashCommands::IssueNew, - Gitlab::SlashCommands::IssueSearch, - Gitlab::SlashCommands::IssueMove, - Gitlab::SlashCommands::Deploy - ].freeze + def self.commands + [ + Gitlab::SlashCommands::IssueShow, + Gitlab::SlashCommands::IssueNew, + Gitlab::SlashCommands::IssueSearch, + Gitlab::SlashCommands::IssueMove, + Gitlab::SlashCommands::Deploy + ] + end def execute command, match = match_command @@ -37,7 +41,7 @@ module Gitlab private def available_commands - COMMANDS.select do |klass| + self.class.commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 93e00ab75a1..157d924f99f 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class Deploy < BaseCommand @@ -36,6 +38,7 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def find_action(from, to) environment = project.environments.find_by(name: from) return unless environment @@ -50,6 +53,7 @@ module Gitlab actions.first end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/help.rb b/lib/gitlab/slash_commands/help.rb index 81f3707e03e..dbe15baa3d7 100644 --- a/lib/gitlab/slash_commands/help.rb +++ b/lib/gitlab/slash_commands/help.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class Help < BaseCommand diff --git a/lib/gitlab/slash_commands/issue_command.rb b/lib/gitlab/slash_commands/issue_command.rb index 3d96982b820..4c8dc4b1784 100644 --- a/lib/gitlab/slash_commands/issue_command.rb +++ b/lib/gitlab/slash_commands/issue_command.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class IssueCommand < BaseCommand diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index 3985e635983..d2f1f130b38 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class IssueMove < IssueCommand diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 25f965e843d..48379031537 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class IssueNew < IssueCommand def self.match(text) # we can not match \n with the dot by passing the m modifier as than - # the title and description are not seperated + # the title and description are not separated /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end diff --git a/lib/gitlab/slash_commands/issue_search.rb b/lib/gitlab/slash_commands/issue_search.rb index acba84b54b4..0a705de4484 100644 --- a/lib/gitlab/slash_commands/issue_search.rb +++ b/lib/gitlab/slash_commands/issue_search.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class IssueSearch < IssueCommand @@ -9,6 +11,7 @@ module Gitlab "issue search <your query>" end + # rubocop: disable CodeReuse/ActiveRecord def execute(match) issues = collection.search(match[:query]).limit(QUERY_LIMIT) @@ -18,6 +21,7 @@ module Gitlab Presenters::Access.new(issues).not_found end end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/slash_commands/issue_show.rb b/lib/gitlab/slash_commands/issue_show.rb index ffa5184e5cb..5f5fa32ff20 100644 --- a/lib/gitlab/slash_commands/issue_show.rb +++ b/lib/gitlab/slash_commands/issue_show.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands class IssueShow < IssueCommand diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 1a817eb735b..fa163cb098e 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters @@ -15,7 +17,7 @@ module Gitlab if @resource ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" + ":sweat_smile: Couldn't identify you, nor can I authorize you!" end ephemeral_response(text: message) diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index e13808a2720..73814aa180f 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/deploy.rb b/lib/gitlab/slash_commands/presenters/deploy.rb index ebae0f57f9b..7d852eb1f9a 100644 --- a/lib/gitlab/slash_commands/presenters/deploy.rb +++ b/lib/gitlab/slash_commands/presenters/deploy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index ab855319077..480d7aa6a30 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 31c1e97efba..b6db103b82b 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/issue_move.rb b/lib/gitlab/slash_commands/presenters/issue_move.rb index 03921729941..ca0644ede95 100644 --- a/lib/gitlab/slash_commands/presenters/issue_move.rb +++ b/lib/gitlab/slash_commands/presenters/issue_move.rb @@ -1,4 +1,6 @@ # coding: utf-8 +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 5964bfe9960..ac78745ae70 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb index 4e27d668685..0d497efec0e 100644 --- a/lib/gitlab/slash_commands/presenters/issue_search.rb +++ b/lib/gitlab/slash_commands/presenters/issue_search.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb index 562f15f403c..5a2c79a928e 100644 --- a/lib/gitlab/slash_commands/presenters/issue_show.rb +++ b/lib/gitlab/slash_commands/presenters/issue_show.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SlashCommands module Presenters @@ -38,10 +40,10 @@ module Gitlab end def text - message = "**#{status_text(@resource)}**" + message = ["**#{status_text(@resource)}**"] if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero? - return message + return message.join end message << " · " @@ -49,7 +51,7 @@ module Gitlab message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero? message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero? - message + message.join end def pretext diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 3669dedf0fe..607c9c8dec1 100644 --- a/lib/gitlab/slash_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,4 +1,7 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module SlashCommands Result = Struct.new(:type, :message) end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 4f86b3e8f73..e360b552f89 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class SnippetSearchResults < SearchResults include SnippetsHelper @@ -30,13 +32,17 @@ module Gitlab private + # rubocop: disable CodeReuse/ActiveRecord def snippet_titles limit_snippets.search(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def snippet_blobs limit_snippets.search_code(query).order('updated_at DESC').includes(:author) end + # rubocop: enable CodeReuse/ActiveRecord def default_scope 'snippet_blobs' diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb new file mode 100644 index 00000000000..7817a2a1ce2 --- /dev/null +++ b/lib/gitlab/sql/cte.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for easily building CTE statements. + # + # Example: + # + # cte = CTE.new(:my_cte_name) + # ns = Arel::Table.new(:namespaces) + # + # cte << Namespace. + # where(ns[:parent_id].eq(some_namespace_id)) + # + # Namespace + # with(cte.to_arel). + # from(cte.alias_to(ns)) + class CTE + attr_reader :table, :query + + # name - The name of the CTE as a String or Symbol. + def initialize(name, query) + @table = Arel::Table.new(name) + @query = query + end + + # Returns the Arel relation for this CTE. + def to_arel + sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") + + Arel::Nodes::As.new(table, sql) + end + + # Returns an "AS" statement that aliases the CTE name as the given table + # name. This allows one to trick ActiveRecord into thinking it's selecting + # from an actual table, when in reality it's selecting from a CTE. + # + # alias_table - The Arel table to use as the alias. + def alias_to(alias_table) + Arel::Nodes::As.new(table, alias_table) + end + + # Applies the CTE to the given relation, returning a new one that will + # query from it. + def apply_to(relation) + relation.except(:where) + .with(to_arel) + .from(alias_to(relation.model.arel_table)) + end + end + end +end diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb index 5e89e12b2b1..f3421bd95d2 100644 --- a/lib/gitlab/sql/glob.rb +++ b/lib/gitlab/sql/glob.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SQL module Glob diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 53744bad1f4..92388262035 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SQL module Pattern diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index 16ec002f139..ec1f00a3a91 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SQL # Class for easily building recursive CTE statements. diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index c99b262f1ca..f05592fc3a3 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module SQL # Class for building SQL UNION statements. @@ -7,7 +9,7 @@ module Gitlab # # Example usage: # - # union = Gitlab::SQL::Union.new(user.personal_projects, user.projects) + # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) # sql = union.to_sql # # Project.where("id IN (#{sql})") diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 6f63ea91ae8..6df54852d02 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module Gitlab class SSHPublicKey Technology = Struct.new(:name, :key_class, :supported_sizes) - Technologies = [ + TECHNOLOGIES = [ Technology.new(:rsa, OpenSSL::PKey::RSA, [1024, 2048, 3072, 4096]), Technology.new(:dsa, OpenSSL::PKey::DSA, [1024, 2048, 3072]), Technology.new(:ecdsa, OpenSSL::PKey::EC, [256, 384, 521]), @@ -10,11 +12,11 @@ module Gitlab ].freeze def self.technology(name) - Technologies.find { |tech| tech.name.to_s == name.to_s } + TECHNOLOGIES.find { |tech| tech.name.to_s == name.to_s } end def self.technology_for_key(key) - Technologies.find { |tech| key.is_a?(tech.key_class) } + TECHNOLOGIES.find { |tech| key.is_a?(tech.key_class) } end def self.supported_sizes(name) @@ -26,7 +28,7 @@ module Gitlab return key_content if parts.empty? - parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| + parts.each_with_object(+"#{ssh_type} ").with_index do |(part, content), index| content << part if Gitlab::SSHPublicKey.new(content).valid? diff --git a/lib/gitlab/storage_check.rb b/lib/gitlab/storage_check.rb deleted file mode 100644 index fe81513c9ec..00000000000 --- a/lib/gitlab/storage_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -require_relative 'storage_check/cli' -require_relative 'storage_check/gitlab_caller' -require_relative 'storage_check/option_parser' -require_relative 'storage_check/response' - -module Gitlab - module StorageCheck - ENDPOINT = '/-/storage_check'.freeze - Options = Struct.new(:target, :token, :interval, :dryrun) - end -end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb deleted file mode 100644 index 9b64c8e033a..00000000000 --- a/lib/gitlab/storage_check/cli.rb +++ /dev/null @@ -1,71 +0,0 @@ -module Gitlab - module StorageCheck - class CLI - def self.start!(args) - runner = new(Gitlab::StorageCheck::OptionParser.parse!(args)) - runner.start_loop - end - - attr_reader :logger, :options - - def initialize(options) - @options = options - @logger = Logger.new(STDOUT) - end - - def start_loop - logger.info "Checking #{options.target} every #{options.interval} seconds" - - if options.dryrun - logger.info "Dryrun, exiting..." - return - end - - begin - loop do - response = GitlabCaller.new(options).call! - log_response(response) - update_settings(response) - - sleep options.interval - end - rescue Interrupt - logger.info "Ending storage-check" - end - end - - def update_settings(response) - previous_interval = options.interval - - if response.valid? - options.interval = response.check_interval || previous_interval - end - - if previous_interval != options.interval - logger.info "Interval changed: #{options.interval} seconds" - end - end - - def log_response(response) - unless response.valid? - return logger.error("Invalid response checking nfs storage: #{response.http_response.inspect}") - end - - if response.responsive_shards.any? - logger.debug("Responsive shards: #{response.responsive_shards.join(', ')}") - end - - warnings = [] - if response.skipped_shards.any? - warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" - end - - if response.failing_shards.any? - warnings << "Failing shards: #{response.failing_shards.join(', ')}" - end - - logger.warn(warnings.join(' - ')) if warnings.any? - end - end - end -end diff --git a/lib/gitlab/storage_check/gitlab_caller.rb b/lib/gitlab/storage_check/gitlab_caller.rb deleted file mode 100644 index 44952b68844..00000000000 --- a/lib/gitlab/storage_check/gitlab_caller.rb +++ /dev/null @@ -1,39 +0,0 @@ -require 'excon' - -module Gitlab - module StorageCheck - class GitlabCaller - def initialize(options) - @options = options - end - - def call! - Gitlab::StorageCheck::Response.new(get_response) - rescue Errno::ECONNREFUSED, Excon::Error - # Server not ready, treated as invalid response. - Gitlab::StorageCheck::Response.new(nil) - end - - def get_response - scheme, *other_parts = URI.split(@options.target) - socket_path = if scheme == 'unix' - other_parts.compact.join - end - - connection = Excon.new(@options.target, socket: socket_path) - connection.post(path: Gitlab::StorageCheck::ENDPOINT, - headers: headers) - end - - def headers - @headers ||= begin - headers = {} - headers['Content-Type'] = headers['Accept'] = 'application/json' - headers['TOKEN'] = @options.token if @options.token - - headers - end - end - end - end -end diff --git a/lib/gitlab/storage_check/option_parser.rb b/lib/gitlab/storage_check/option_parser.rb deleted file mode 100644 index 66ed7906f97..00000000000 --- a/lib/gitlab/storage_check/option_parser.rb +++ /dev/null @@ -1,39 +0,0 @@ -module Gitlab - module StorageCheck - class OptionParser - def self.parse!(args) - # Start out with some defaults - options = Gitlab::StorageCheck::Options.new(nil, nil, 1, false) - - parser = ::OptionParser.new do |opts| - opts.banner = "Usage: bin/storage_check [options]" - - opts.on('-t=string', '--target string', 'URL or socket to trigger storage check') do |value| - options.target = value - end - - opts.on('-T=string', '--token string', 'Health token to use') { |value| options.token = value } - - opts.on('-i=n', '--interval n', ::OptionParser::DecimalInteger, 'Seconds between checks') do |value| - options.interval = value - end - - opts.on('-d', '--dryrun', "Output what will be performed, but don't start the process") do |value| - options.dryrun = value - end - end - parser.parse!(args) - - unless options.target - raise ::OptionParser::InvalidArgument.new('Provide a URI to provide checks') - end - - if URI.parse(options.target).scheme.nil? - raise ::OptionParser::InvalidArgument.new('Add the scheme to the target, `unix://`, `https://` or `http://` are supported') - end - - options - end - end - end -end diff --git a/lib/gitlab/storage_check/response.rb b/lib/gitlab/storage_check/response.rb deleted file mode 100644 index 326ab236e3e..00000000000 --- a/lib/gitlab/storage_check/response.rb +++ /dev/null @@ -1,77 +0,0 @@ -require 'json' - -module Gitlab - module StorageCheck - class Response - attr_reader :http_response - - def initialize(http_response) - @http_response = http_response - end - - def valid? - @http_response && (200...299).cover?(@http_response.status) && - @http_response.headers['Content-Type'].include?('application/json') && - parsed_response - end - - def check_interval - return nil unless parsed_response - - parsed_response['check_interval'] - end - - def responsive_shards - divided_results[:responsive_shards] - end - - def skipped_shards - divided_results[:skipped_shards] - end - - def failing_shards - divided_results[:failing_shards] - end - - private - - def results - return [] unless parsed_response - - parsed_response['results'] - end - - def divided_results - return @divided_results if @divided_results - - @divided_results = {} - @divided_results[:responsive_shards] = [] - @divided_results[:skipped_shards] = [] - @divided_results[:failing_shards] = [] - - results.each do |info| - name = info['storage'] - - case info['success'] - when true - @divided_results[:responsive_shards] << name - when false - @divided_results[:failing_shards] << name - else - @divided_results[:skipped_shards] << name - end - end - - @divided_results - end - - def parsed_response - return @parsed_response if defined?(@parsed_response) - - @parsed_response = JSON.parse(@http_response.body) - rescue JSON::JSONError - @parsed_response = nil - end - end - end -end diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb index 9a2219b7d77..62621255a53 100644 --- a/lib/gitlab/string_placeholder_replacer.rb +++ b/lib/gitlab/string_placeholder_replacer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class StringPlaceholderReplacer # This method accepts the following paras diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index c6ad997a4d4..780fe4c7725 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class StringRangeMarker attr_accessor :raw_line, :rich_line, :html_escaped diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb index b19aa6dea35..f1982ff914c 100644 --- a/lib/gitlab/string_regex_marker.rb +++ b/lib/gitlab/string_regex_marker.rb @@ -1,5 +1,8 @@ +# frozen_string_literal: true + module Gitlab class StringRegexMarker < StringRangeMarker + # rubocop: disable CodeReuse/ActiveRecord def mark(regex, group: 0, &block) ranges = [] @@ -11,5 +14,6 @@ module Gitlab super(ranges, &block) end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 42be301fd9b..224bb648d8f 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'rainbow/ext/string' require 'gitlab/utils/strong_memoize' @@ -39,7 +41,7 @@ module Gitlab File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] end - os_name.try(:squish!) + os_name.try(:squish) end # Prompt the user to input something @@ -128,17 +130,21 @@ module Gitlab end def all_repos - Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each_value do |repository_storage| + IO.popen(%W(find #{repository_storage.legacy_disk_path} -mindepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end end def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } + end end def user_home diff --git a/lib/gitlab/tcp_checker.rb b/lib/gitlab/tcp_checker.rb index 6e24e46d0ea..f37a044b607 100644 --- a/lib/gitlab/tcp_checker.rb +++ b/lib/gitlab/tcp_checker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class TcpChecker attr_reader :remote_host, :remote_port, :local_host, :local_port, :error diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 7393574ac13..0b4cc571dc0 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,21 +1,34 @@ +# frozen_string_literal: true + module Gitlab module Template class BaseTemplate - def initialize(path, project = nil) + attr_accessor :category + + def initialize(path, project = nil, category: nil) @path = path + @category = category @finder = self.class.finder(project) end def name File.basename(@path, self.class.extension) end + alias_method :key, :name def content @finder.read(@path) end + # Present for compatibility with license templates, which can replace text + # like `[fullname]` with a user-specified string. This is a no-op for + # other templates + def resolve!(_placeholders = {}) + self + end + def to_json - { name: name, content: content } + { key: key, name: name, content: content } end def <=>(other) @@ -62,7 +75,7 @@ module Gitlab directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project) }.sort + files.map { |f| new(f, project, category: category) }.sort end def category_directory(category) diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index 20b054b0bd8..3b516bb862a 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template class DockerfileTemplate < BaseTemplate diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb index 473b05257c6..93c229af143 100644 --- a/lib/gitlab/template/finders/base_template_finder.rb +++ b/lib/gitlab/template/finders/base_template_finder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template module Finders @@ -21,7 +23,7 @@ module Gitlab def category_directory(category) return @base_dir unless category.present? - @base_dir + @categories[category] + File.join(@base_dir, @categories[category]) end class << self diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 831da45191f..2dd4b7a4092 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -1,4 +1,6 @@ -# Searches and reads file present on Gitlab installation directory +# frozen_string_literal: true + +# Searches and reads file present on GitLab installation directory module Gitlab module Template module Finders @@ -16,12 +18,16 @@ module Gitlab def find(key) file_name = "#{key}#{@extension}" + # The key is untrusted input, so ensure we can't be directed outside + # of base_dir + Gitlab::Utils.check_path_traversal!(file_name) + directory = select_directory(file_name) directory ? File.join(category_directory(directory), file_name) : nil end def list_files_for(dir) - dir << '/' unless dir.end_with?('/') + dir = "#{dir}/" unless dir.end_with?('/') Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) } end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 33f07fa0120..8e234148a63 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -1,4 +1,6 @@ -# Searches and reads files present on each Gitlab project repository +# frozen_string_literal: true + +# Searches and reads files present on each GitLab project repository module Gitlab module Template module Finders @@ -24,21 +26,26 @@ module Gitlab def find(key) file_name = "#{key}#{@extension}" + + # The key is untrusted input, so ensure we can't be directed outside + # of base_dir inside the repository + Gitlab::Utils.check_path_traversal!(file_name) + directory = select_directory(file_name) raise FileNotFoundError if directory.nil? - category_directory(directory) + file_name + File.join(category_directory(directory), file_name) end def list_files_for(dir) return [] unless @commit - dir << '/' unless dir.end_with?('/') + dir = "#{dir}/" unless dir.end_with?('/') entries = @repository.tree(:head, dir).entries - names = entries.map(&:name) - names.select { |f| f =~ self.class.filter_regex(@extension) } + paths = entries.map(&:path) + paths.select { |f| f =~ self.class.filter_regex(@extension) } end private @@ -47,10 +54,10 @@ module Gitlab return [] unless @commit # Insert root as directory - directories = ["", @categories.keys] + directories = ["", *@categories.keys] directories.find do |category| - path = category_directory(category) + file_name + path = File.join(category_directory(category), file_name) @repository.blob_at(@commit.id, path) end end diff --git a/lib/gitlab/template/gitignore_template.rb b/lib/gitlab/template/gitignore_template.rb index 8d2a9d2305c..72a1b7460c2 100644 --- a/lib/gitlab/template/gitignore_template.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template class GitignoreTemplate < BaseTemplate diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index fd040148a1e..fbefb5f7f0e 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template class GitlabCiYmlTemplate < BaseTemplate @@ -20,7 +22,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/gitlab-ci-yml') + Rails.root.join('lib/gitlab/ci/templates') end def finder(project = nil) diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb index c6fa8d3eafc..01b191733d4 100644 --- a/lib/gitlab/template/issue_template.rb +++ b/lib/gitlab/template/issue_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template class IssueTemplate < BaseTemplate diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb index f826c02f3b5..357b31cd82e 100644 --- a/lib/gitlab/template/merge_request_template.rb +++ b/lib/gitlab/template/merge_request_template.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Template class MergeRequestTemplate < BaseTemplate diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb new file mode 100644 index 00000000000..b0e01697a66 --- /dev/null +++ b/lib/gitlab/template_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module TemplateHelper + def prepare_template_environment(file) + return unless file + + params[:import_export_upload] = ImportExportUpload.new(import_file: file) + end + + def tmp_filename + SecureRandom.hex + end + end +end diff --git a/lib/gitlab/temporarily_allow.rb b/lib/gitlab/temporarily_allow.rb index 880e55f71df..000f8ca699d 100644 --- a/lib/gitlab/temporarily_allow.rb +++ b/lib/gitlab/temporarily_allow.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module TemporarilyAllow TEMPORARILY_ALLOW_MUTEX = Mutex.new @@ -10,7 +12,7 @@ module Gitlab end def temporarily_allowed?(key) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] > 0 else TEMPORARILY_ALLOW_MUTEX.synchronize do @@ -26,11 +28,11 @@ module Gitlab end def temporarily_allow_request_store - RequestStore[:temporarily_allow] ||= Hash.new(0) + Gitlab::SafeRequestStore[:temporarily_allow] ||= Hash.new(0) end def temporarily_allow_add(key, value) - if RequestStore.active? + if Gitlab::SafeRequestStore.active? temporarily_allow_request_store[key] += value else TEMPORARILY_ALLOW_MUTEX.synchronize do diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index 53333b9b06b..513cbe839ba 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable Style/ClassVars # This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests diff --git a/lib/gitlab/testing/request_inspector_middleware.rb b/lib/gitlab/testing/request_inspector_middleware.rb index e387667480d..36cdfebcc28 100644 --- a/lib/gitlab/testing/request_inspector_middleware.rb +++ b/lib/gitlab/testing/request_inspector_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # rubocop:disable Style/ClassVars module Gitlab @@ -35,11 +37,15 @@ module Gitlab request_headers = env_http_headers(env) status, headers, body = @app.call(env) + full_body = +'' + body.each { |b| full_body << b } + request = OpenStruct.new( url: url, status_code: status, request_headers: request_headers, - response_headers: headers + response_headers: headers, + body: full_body ) log_request request diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index d43eff5ba4a..63860b9cb26 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # Module containing GitLab's application theme definitions and helper methods # for accessing them. @@ -12,11 +14,16 @@ module Gitlab # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui_indigo'), - Theme.new(2, 'Dark', 'ui_dark'), - Theme.new(3, 'Light', 'ui_light'), - Theme.new(4, 'Blue', 'ui_blue'), - Theme.new(5, 'Green', 'ui_green') + Theme.new(1, 'Indigo', 'ui-indigo'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo'), + Theme.new(4, 'Blue', 'ui-blue'), + Theme.new(7, 'Light Blue', 'ui-light-blue'), + Theme.new(5, 'Green', 'ui-green'), + Theme.new(8, 'Light Green', 'ui-light-green'), + Theme.new(9, 'Red', 'ui-red'), + Theme.new(10, 'Light Red', 'ui-light-red'), + Theme.new(2, 'Dark', 'ui-dark'), + Theme.new(3, 'Light', 'ui-light') ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index d615c24149a..cc206010e74 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module TimeTrackingFormatter extend self diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index 76a1808c8ac..4f974c98c71 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Timeless def self.timeless(model, &block) diff --git a/lib/gitlab/tracing.rb b/lib/gitlab/tracing.rb new file mode 100644 index 00000000000..3c4db42ac06 --- /dev/null +++ b/lib/gitlab/tracing.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + # Only enable tracing when the `GITLAB_TRACING` env var is configured. Note that we avoid using ApplicationSettings since + # the same environment variable needs to be configured for Workhorse, Gitaly and any other components which + # emit tracing. Since other components may start before Rails, and may not have access to ApplicationSettings, + # an env var makes more sense. + def self.enabled? + connection_string.present? + end + + def self.connection_string + ENV['GITLAB_TRACING'] + end + end +end diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb new file mode 100644 index 00000000000..3a08ede8138 --- /dev/null +++ b/lib/gitlab/tracing/common.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Common + def tracer + OpenTracing.global_tracer + end + + # Convience method for running a block with a span + def in_tracing_span(operation_name:, tags:, child_of: nil) + scope = tracer.start_active_span( + operation_name, + child_of: child_of, + tags: tags + ) + span = scope.span + + # Add correlation details to the span if we have them + correlation_id = Gitlab::CorrelationId.current_id + if correlation_id + span.set_tag('correlation_id', correlation_id) + end + + begin + yield span + rescue => e + log_exception_on_span(span, e) + raise e + ensure + scope.close + end + end + + def postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil) + span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of) + + log_exception_on_span(span, exception) if exception + + span.finish(end_time: end_time) + end + + def log_exception_on_span(span, exception) + span.set_tag('error', true) + span.log_kv(kv_tags_for_exception(exception)) + end + + def kv_tags_for_exception(exception) + case exception + when Exception + { + 'event': 'error', + 'error.kind': exception.class.to_s, + 'message': Gitlab::UrlSanitizer.sanitize(exception.message), + 'stack': exception.backtrace&.join("\n") + } + else + { + 'event': 'error', + 'error.kind': exception.class.to_s, + 'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s) + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/factory.rb b/lib/gitlab/tracing/factory.rb new file mode 100644 index 00000000000..fc714164353 --- /dev/null +++ b/lib/gitlab/tracing/factory.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require "cgi" + +module Gitlab + module Tracing + class Factory + OPENTRACING_SCHEME = "opentracing" + + def self.create_tracer(service_name, connection_string) + return unless connection_string.present? + + begin + opentracing_details = parse_connection_string(connection_string) + driver_name = opentracing_details[:driver_name] + + case driver_name + when "jaeger" + JaegerFactory.create_tracer(service_name, opentracing_details[:options]) + else + raise "Unknown driver: #{driver_name}" + end + rescue => e + # Can't create the tracer? Warn and continue sans tracer + warn "Unable to instantiate tracer: #{e}" + nil + end + end + + def self.parse_connection_string(connection_string) + parsed = URI.parse(connection_string) + + unless valid_uri?(parsed) + raise "Invalid tracing connection string" + end + + { + driver_name: parsed.host, + options: parse_query(parsed.query) + } + end + private_class_method :parse_connection_string + + def self.parse_query(query) + return {} unless query + + CGI.parse(query).symbolize_keys.transform_values(&:first) + end + private_class_method :parse_query + + def self.valid_uri?(uri) + return false unless uri + + uri.scheme == OPENTRACING_SCHEME && + uri.host.to_s =~ /^[a-z0-9_]+$/ && + uri.path.empty? + end + private_class_method :valid_uri? + end + end +end diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb new file mode 100644 index 00000000000..6c2aab73125 --- /dev/null +++ b/lib/gitlab/tracing/grpc_interceptor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'opentracing' +require 'grpc' + +module Gitlab + module Tracing + class GRPCInterceptor < GRPC::ClientInterceptor + include Common + include Singleton + + def request_response(request:, call:, method:, metadata:) + wrap_with_tracing(method, 'unary', metadata) do + yield + end + end + + def client_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, 'client_stream', metadata) do + yield + end + end + + def server_streamer(request:, call:, method:, metadata:) + wrap_with_tracing(method, 'server_stream', metadata) do + yield + end + end + + def bidi_streamer(requests:, call:, method:, metadata:) + wrap_with_tracing(method, 'bidi_stream', metadata) do + yield + end + end + + private + + def wrap_with_tracing(method, grpc_type, metadata) + tags = { + 'component' => 'grpc', + 'span.kind' => 'client', + 'grpc.method' => method, + 'grpc.type' => grpc_type + } + + in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span| + OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata) + + yield + end + end + end + end +end diff --git a/lib/gitlab/tracing/jaeger_factory.rb b/lib/gitlab/tracing/jaeger_factory.rb new file mode 100644 index 00000000000..2682007302a --- /dev/null +++ b/lib/gitlab/tracing/jaeger_factory.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'jaeger/client' + +module Gitlab + module Tracing + class JaegerFactory + # When the probabilistic sampler is used, by default 0.1% of requests will be traced + DEFAULT_PROBABILISTIC_RATE = 0.001 + + # The default port for the Jaeger agent UDP listener + DEFAULT_UDP_PORT = 6831 + + # Reduce this from default of 10 seconds as the Ruby jaeger + # client doesn't have overflow control, leading to very large + # messages which fail to send over UDP (max packet = 64k) + # Flush more often, with smaller packets + FLUSH_INTERVAL = 5 + + def self.create_tracer(service_name, options) + kwargs = { + service_name: service_name, + sampler: get_sampler(options[:sampler], options[:sampler_param]), + reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]) + }.compact + + extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord + if extra_params.present? + message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}" + + if options[:strict_parsing] + raise message + else + warn message + end + end + + Jaeger::Client.build(kwargs) + end + + def self.get_sampler(sampler_type, sampler_param) + case sampler_type + when "probabilistic" + sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE + Jaeger::Samplers::Probabilistic.new(rate: sampler_rate) + when "const" + const_value = sampler_param == "1" + Jaeger::Samplers::Const.new(const_value) + else + nil + end + end + private_class_method :get_sampler + + def self.get_reporter(service_name, http_endpoint, udp_endpoint) + encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name) + + if http_endpoint.present? + sender = get_http_sender(encoder, http_endpoint) + elsif udp_endpoint.present? + sender = get_udp_sender(encoder, udp_endpoint) + else + return nil + end + + Jaeger::Reporters::RemoteReporter.new( + sender: sender, + flush_interval: FLUSH_INTERVAL + ) + end + private_class_method :get_reporter + + def self.get_http_sender(encoder, address) + Jaeger::HttpSender.new( + url: address, + encoder: encoder, + logger: Logger.new(STDOUT) + ) + end + private_class_method :get_http_sender + + def self.get_udp_sender(encoder, address) + pair = address.split(":", 2) + host = pair[0] + port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT + + Jaeger::UdpSender.new( + host: host, + port: port, + encoder: encoder, + logger: Logger.new(STDOUT) + ) + end + private_class_method :get_udp_sender + end + end +end diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb new file mode 100644 index 00000000000..e6a31293f7b --- /dev/null +++ b/lib/gitlab/tracing/rack_middleware.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + class RackMiddleware + include Common + + REQUEST_METHOD = 'REQUEST_METHOD' + + def initialize(app) + @app = app + end + + def call(env) + method = env[REQUEST_METHOD] + + context = tracer.extract(OpenTracing::FORMAT_RACK, env) + tags = { + 'component' => 'rack', + 'span.kind' => 'server', + 'http.method' => method, + 'http.url' => self.class.build_sanitized_url_from_env(env) + } + + in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span| + @app.call(env).tap do |status_code, _headers, _body| + span.set_tag('http.status_code', status_code) + end + end + end + + # Generate a sanitized (safe) request URL from the rack environment + def self.build_sanitized_url_from_env(env) + request = ActionDispatch::Request.new(env) + + original_url = request.original_url + uri = URI.parse(original_url) + uri.query = request.filtered_parameters.to_query if uri.query.present? + + uri.to_s + end + end + end +end diff --git a/lib/gitlab/tracing/rails/action_view_subscriber.rb b/lib/gitlab/tracing/rails/action_view_subscriber.rb new file mode 100644 index 00000000000..88816e1fb32 --- /dev/null +++ b/lib/gitlab/tracing/rails/action_view_subscriber.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + class ActionViewSubscriber + include RailsCommon + + COMPONENT_TAG = 'ActionView' + RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view' + RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view' + RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view' + + # Instruments Rails ActionView events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscriptions = [ + ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_template(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_collection(start, finish, payload) + end, + ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify_render_partial(start, finish, payload) + end + ] + + create_unsubscriber subscriptions + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify_render_template(start, finish, payload) + generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload)) + end + + def notify_render_collection(start, finish, payload) + generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload)) + end + + def notify_render_partial(start, finish, payload) + generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload)) + end + + private + + def tags_for_render_template(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier], + 'template.layout' => payload[:layout] + } + end + + def tags_for_render_collection(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier], + 'template.count' => payload[:count] || 0, + 'template.cache.hits' => payload[:cache_hits] || 0 + } + end + + def tags_for_render_partial(payload) + { + 'component' => COMPONENT_TAG, + 'template.id' => payload[:identifier] + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/rails/active_record_subscriber.rb b/lib/gitlab/tracing/rails/active_record_subscriber.rb new file mode 100644 index 00000000000..32f5658e57e --- /dev/null +++ b/lib/gitlab/tracing/rails/active_record_subscriber.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + class ActiveRecordSubscriber + include RailsCommon + + ACTIVE_RECORD_NOTIFICATION_TOPIC = 'sql.active_record' + OPERATION_NAME_PREFIX = 'active_record:' + DEFAULT_OPERATION_NAME = 'sqlquery' + + # Instruments Rails ActiveRecord events for opentracing. + # Returns a lambda, which, when called will unsubscribe from the notifications + def self.instrument + subscriber = new + + subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| + subscriber.notify(start, finish, payload) + end + + create_unsubscriber [subscription] + end + + # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html + def notify(start, finish, payload) + generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload)) + end + + private + + def notification_name(payload) + OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME) + end + + def tags_for_notification(payload) + { + 'component' => 'ActiveRecord', + 'span.kind' => 'client', + 'db.type' => 'sql', + 'db.connection_id' => payload[:connection_id], + 'db.cached' => payload[:cached] || false, + 'db.statement' => payload[:sql] + } + end + end + end + end +end diff --git a/lib/gitlab/tracing/rails/rails_common.rb b/lib/gitlab/tracing/rails/rails_common.rb new file mode 100644 index 00000000000..88e914f62f8 --- /dev/null +++ b/lib/gitlab/tracing/rails/rails_common.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Rails + module RailsCommon + extend ActiveSupport::Concern + include Gitlab::Tracing::Common + + class_methods do + def create_unsubscriber(subscriptions) + -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } } + end + end + + def generate_span_for_notification(operation_name, start, finish, payload, tags) + exception = payload[:exception] + + postnotify_span(operation_name, start, finish, tags: tags, exception: exception) + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb new file mode 100644 index 00000000000..2b71c1ea21e --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/client_middleware.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Sidekiq + class ClientMiddleware + include SidekiqCommon + + SPAN_KIND = 'client' + + def call(worker_class, job, queue, redis_pool) + in_tracing_span( + operation_name: "sidekiq:#{job['class']}", + tags: tags_from_job(job, SPAN_KIND)) do |span| + # Inject the details directly into the job + tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job) + + yield + end + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb new file mode 100644 index 00000000000..5b43c4310e6 --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/server_middleware.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'opentracing' + +module Gitlab + module Tracing + module Sidekiq + class ServerMiddleware + include SidekiqCommon + + SPAN_KIND = 'server' + + def call(worker, job, queue) + context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job) + + in_tracing_span( + operation_name: "sidekiq:#{job['class']}", + child_of: context, + tags: tags_from_job(job, SPAN_KIND)) do |span| + yield + end + end + end + end + end +end diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb new file mode 100644 index 00000000000..a911a29d773 --- /dev/null +++ b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Tracing + module Sidekiq + module SidekiqCommon + include Gitlab::Tracing::Common + + def tags_from_job(job, kind) + { + 'component' => 'sidekiq', + 'span.kind' => kind, + 'sidekiq.queue' => job['queue'], + 'sidekiq.jid' => job['jid'], + 'sidekiq.retry' => job['retry'].to_s, + 'sidekiq.args' => job['args']&.join(", ") + } + end + end + end + end +end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb new file mode 100644 index 00000000000..453d78e2f7b --- /dev/null +++ b/lib/gitlab/tree_summary.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +module Gitlab + class TreeSummary + include ::Gitlab::Utils::StrongMemoize + + attr_reader :commit, :project, :path, :offset, :limit + + attr_reader :resolved_commits + private :resolved_commits + + def initialize(commit, project, params = {}) + @commit = commit + @project = project + + @path = params.fetch(:path, nil).presence + @offset = params.fetch(:offset, 0).to_i + @limit = (params.fetch(:limit, 25) || 25).to_i + + # Ensure that if multiple tree entries share the same last commit, they share + # a ::Commit instance. This prevents us from rendering the same commit title + # multiple times + @resolved_commits = {} + end + + # Creates a summary of the tree entries for a commit, within the window of + # entries defined by the offset and limit parameters. This consists of two + # return values: + # + # - An Array of Hashes containing the following keys: + # - file_name: The full path of the tree entry + # - type: One of :blob, :tree, or :submodule + # - commit: The last ::Commit to touch this entry in the tree + # - commit_path: URI of the commit in the web interface + # - An Array of the unique ::Commit objects in the first value + def summarize + summary = contents + .map { |content| build_entry(content) } + .tap { |summary| fill_last_commits!(summary) } + + [summary, commits] + end + + # Does the tree contain more entries after the given offset + limit? + def more? + all_contents[next_offset].present? + end + + # The offset of the next batch of tree entries. If more? returns false, this + # batch will be empty + def next_offset + [all_contents.size + 1, offset + limit].min + end + + private + + def contents + all_contents[offset, limit] + end + + def commits + resolved_commits.values + end + + def repository + project.repository + end + + def entry_path(entry) + File.join(*[path, entry[:file_name]].compact) + end + + def build_entry(entry) + { file_name: entry.name, type: entry.type } + end + + def fill_last_commits!(entries) + # Ensure the path is in "path/" format + ensured_path = + if path + File.join(*[path, ""]) + end + + commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit) + + entries.each do |entry| + path_key = entry_path(entry) + commit = cache_commit(commits_hsh[path_key]) + + if commit + entry[:commit] = commit + entry[:commit_path] = commit_path(commit) + end + end + end + + def cache_commit(commit) + return nil unless commit.present? + + resolved_commits[commit.id] ||= commit + end + + def commit_path(commit) + Gitlab::Routing.url_helpers.project_commit_path(project, commit) + end + + def all_contents + strong_memoize(:all_contents) do + [ + *tree.trees, + *tree.blobs, + *tree.submodules + ] + end + end + + def tree + strong_memoize(:tree) { repository.tree(commit.id, path) } + end + end +end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index dc2d91dfa23..ba1137313d8 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab # An untrusted regular expression is any regexp containing patterns sourced # from user input. diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb index 8947ecfb92e..bc066bf4143 100644 --- a/lib/gitlab/update_path_error.rb +++ b/lib/gitlab/update_path_error.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab UpdatePathError = Class.new(StandardError) end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb deleted file mode 100644 index 024be6aca44..00000000000 --- a/lib/gitlab/upgrader.rb +++ /dev/null @@ -1,109 +0,0 @@ -module Gitlab - class Upgrader - def execute - puts "GitLab #{current_version.major} upgrade tool" - puts "Your version is #{current_version}" - puts "Latest available version for GitLab #{current_version.major} is #{latest_version}" - - if latest_version? - puts "You are using the latest GitLab version" - else - puts "Newer GitLab version is available" - - answer = if ARGV.first == "-y" - "yes" - else - prompt("Do you want to upgrade (yes/no)? ", %w{yes no}) - end - - if answer == "yes" - upgrade - else - exit 0 - end - end - end - - def latest_version? - current_version >= latest_version - end - - def current_version - @current_version ||= Gitlab::VersionInfo.parse(current_version_raw) - end - - def latest_version - @latest_version ||= Gitlab::VersionInfo.parse(latest_version_raw) - end - - def current_version_raw - File.read(File.join(gitlab_path, "VERSION")).strip - end - - def latest_version_raw - git_tags = fetch_git_tags - git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ } - git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) } - "v#{git_versions.sort.last}" - end - - def fetch_git_tags - remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) - end - - def update_commands - { - "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), - "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), - "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), - "Install gems" => %w(bundle), - "Migrate DB" => %w(bundle exec rake db:migrate), - "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile), - "Clear cache" => %w(bundle exec rake cache:clear) - } - end - - def env - { - 'RAILS_ENV' => 'production', - 'NODE_ENV' => 'production' - } - end - - def upgrade - update_commands.each do |title, cmd| - puts title - puts " -> #{cmd.join(' ')}" - - if system(env, *cmd) - puts " -> OK" - else - puts " -> FAILED" - puts "Failed to upgrade. Try to repeat task or proceed with upgrade manually " - exit 1 - end - end - - puts "Done" - end - - def gitlab_path - File.expand_path(File.join(File.dirname(__FILE__), '../..')) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while !choices.include?(answer) - answer - end - end -end diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index 7d7400bdabf..e0e7084e27e 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class UploadsTransfer < ProjectTransfer def root_dir diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 20be193ea0c..9b7b0db9525 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -1,36 +1,43 @@ +# frozen_string_literal: true + require 'resolv' +require 'ipaddress' module Gitlab class UrlBlocker BlockedUrlError = Class.new(StandardError) class << self - def validate!(url, allow_localhost: false, allow_local_network: true, ports: [], protocols: []) + def validate!(url, ports: [], protocols: [], allow_localhost: false, allow_local_network: true, ascii_only: false, enforce_user: false, enforce_sanitization: false) return true if url.nil? - begin - uri = Addressable::URI.parse(url) - rescue Addressable::URI::InvalidURIError - raise BlockedUrlError, "URI is invalid" - end + # Param url can be a string, URI or Addressable::URI + uri = parse_url(url) + + validate_html_tags!(uri) if enforce_sanitization # Allow imports from the GitLab instance itself but only from the configured ports return true if internal?(uri) - port = uri.port || uri.default_port + port = get_port(uri) validate_protocol!(uri.scheme, protocols) validate_port!(port, ports) if ports.any? - validate_user!(uri.user) + validate_user!(uri.user) if enforce_user validate_hostname!(uri.hostname) + validate_unicode_restriction!(uri) if ascii_only begin - addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end rescue SocketError return true end validate_localhost!(addrs_info) unless allow_localhost + validate_loopback!(addrs_info) unless allow_localhost validate_local_network!(addrs_info) unless allow_local_network + validate_link_local!(addrs_info) unless allow_local_network true end @@ -45,6 +52,30 @@ module Gitlab private + def get_port(uri) + uri.port || uri.default_port + end + + def validate_html_tags!(uri) + uri_str = uri.to_s + sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) + if sanitized_uri != uri_str + raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed' + end + end + + def parse_url(url) + raise Addressable::URI::InvalidURIError if multiline?(url) + + Addressable::URI.parse(url) + rescue Addressable::URI::InvalidURIError, URI::InvalidURIError + raise BlockedUrlError, 'URI is invalid' + end + + def multiline?(url) + CGI.unescape(url.to_s) =~ /\n|\r/ + end + def validate_port!(port, ports) return if port.blank? # Only ports under 1024 are restricted @@ -69,13 +100,20 @@ module Gitlab def validate_hostname!(value) return if value.blank? + return if IPAddress.valid?(value) return if value =~ /\A\p{Alnum}/ - raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + raise BlockedUrlError, "Hostname or IP address invalid" + end + + def validate_unicode_restriction!(uri) + return if uri.to_s.ascii_only? + + raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" end def validate_localhost!(addrs_info) - local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips = ["::", "0.0.0.0"] local_ips.concat(Socket.ip_address_list.map(&:ip_address)) return if (local_ips & addrs_info.map(&:ip_address)).empty? @@ -83,23 +121,38 @@ module Gitlab raise BlockedUrlError, "Requests to localhost are not allowed" end + def validate_loopback!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } + + raise BlockedUrlError, "Requests to loopback addresses are not allowed" + end + def validate_local_network!(addrs_info) - return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } raise BlockedUrlError, "Requests to the local network are not allowed" end + def validate_link_local!(addrs_info) + netmask = IPAddr.new('169.254.0.0/16') + return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } + + raise BlockedUrlError, "Requests to the link local network are not allowed" + end + def internal?(uri) internal_web?(uri) || internal_shell?(uri) end def internal_web?(uri) - uri.hostname == config.gitlab.host && + uri.scheme == config.gitlab.protocol && + uri.hostname == config.gitlab.host && (uri.port.blank? || uri.port == config.gitlab.port) end def internal_shell?(uri) - uri.hostname == config.gitlab_shell.ssh_host && + uri.scheme == 'ssh' && + uri.hostname == config.gitlab_shell.ssh_host && (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 824e2d7251f..f86d599e4cb 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class UrlBuilder include Gitlab::Routing @@ -26,6 +28,8 @@ module Gitlab project_snippet_url(object.project, object) when Snippet snippet_url(object) + when Milestone + milestone_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 59331c827af..880712de5fe 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module Gitlab class UrlSanitizer ALLOWED_SCHEMES = %w[http https ssh git].freeze def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(ALLOWED_SCHEMES) + regexp = URI::DEFAULT_PARSER.make_regexp(ALLOWED_SCHEMES) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -12,6 +14,7 @@ module Gitlab def self.valid?(url) return false unless url.present? + return false unless url.is_a?(String) uri = Addressable::URI.parse(url.strip) @@ -58,7 +61,7 @@ module Gitlab if raw_credentials.present? url.sub!("#{raw_credentials}@", '') - user, password = raw_credentials.split(':') + user, _, password = raw_credentials.partition(':') @credentials ||= { user: user.presence, password: password.presence } end @@ -71,12 +74,10 @@ module Gitlab def generate_full_url return @url unless valid_credentials? - @full_url = @url.dup - - @full_url.password = credentials[:password] if credentials[:password].present? - @full_url.user = credentials[:user] if credentials[:user].present? - - @full_url + @url.dup.tap do |generated| + generated.password = encode_percent(credentials[:password]) if credentials[:password].present? + generated.user = encode_percent(credentials[:user]) if credentials[:user].present? + end end def safe_url @@ -89,5 +90,10 @@ module Gitlab def valid_credentials? credentials && credentials.is_a?(Hash) && credentials.any? end + + def encode_percent(string) + # CGI.escape converts spaces to +, but this doesn't work for git clone + CGI.escape(string).gsub('+', '%20') + end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e294f3c4ebc..6bfcf83f388 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,5 +1,9 @@ +# frozen_string_literal: true + module Gitlab class UsageData + APPROXIMATE_COUNT_MODELS = [Label, MergeRequest, Note, Todo].freeze + class << self def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } @@ -10,6 +14,7 @@ module Gitlab .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) + .merge(usage_counters) end def to_json(force_refresh: false) @@ -21,9 +26,9 @@ module Gitlab uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, - active_user_count: User.active.count, + installation_type: Gitlab::INSTALLATION_TYPE, + active_user_count: count(User.active), recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, edition: 'CE' } @@ -31,54 +36,64 @@ module Gitlab end # rubocop:disable Metrics/AbcSize + # rubocop: disable CodeReuse/ActiveRecord def system_usage_data { counts: { - boards: Board.count, - ci_builds: ::Ci::Build.count, - ci_internal_pipelines: ::Ci::Pipeline.internal.count, - ci_external_pipelines: ::Ci::Pipeline.external.count, - ci_pipeline_config_auto_devops: ::Ci::Pipeline.auto_devops_source.count, - ci_pipeline_config_repository: ::Ci::Pipeline.repository_source.count, - ci_runners: ::Ci::Runner.count, - ci_triggers: ::Ci::Trigger.count, - ci_pipeline_schedules: ::Ci::PipelineSchedule.count, - auto_devops_enabled: ::ProjectAutoDevops.enabled.count, - auto_devops_disabled: ::ProjectAutoDevops.disabled.count, - deploy_keys: DeployKey.count, - deployments: Deployment.count, - environments: ::Environment.count, - clusters: ::Clusters::Cluster.count, - clusters_enabled: ::Clusters::Cluster.enabled.count, - clusters_disabled: ::Clusters::Cluster.disabled.count, - clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, - clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, - clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, - clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, - clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, - clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, - in_review_folder: ::Environment.in_review_folder.count, - groups: Group.count, - issues: Issue.count, - keys: Key.count, - labels: Label.count, - lfs_objects: LfsObject.count, - merge_requests: MergeRequest.count, - milestones: Milestone.count, - notes: Note.count, - pages_domains: PagesDomain.count, - projects: Project.count, - projects_imported_from_github: Project.where(import_type: 'github').count, - protected_branches: ProtectedBranch.count, - releases: Release.count, - remote_mirrors: RemoteMirror.count, - snippets: Snippet.count, - todos: Todo.count, - uploads: Upload.count, - web_hooks: WebHook.count - }.merge(services_usage) + assignee_lists: count(List.assignee), + boards: count(Board), + ci_builds: count(::Ci::Build), + ci_internal_pipelines: count(::Ci::Pipeline.internal), + ci_external_pipelines: count(::Ci::Pipeline.external), + ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), + ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), + ci_runners: count(::Ci::Runner), + ci_triggers: count(::Ci::Trigger), + ci_pipeline_schedules: count(::Ci::PipelineSchedule), + auto_devops_enabled: count(::ProjectAutoDevops.enabled), + auto_devops_disabled: count(::ProjectAutoDevops.disabled), + deploy_keys: count(DeployKey), + deployments: count(Deployment), + environments: count(::Environment), + clusters: count(::Clusters::Cluster), + clusters_enabled: count(::Clusters::Cluster.enabled), + project_clusters_enabled: count(::Clusters::Cluster.enabled.project_type), + group_clusters_enabled: count(::Clusters::Cluster.enabled.group_type), + clusters_disabled: count(::Clusters::Cluster.disabled), + project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), + group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), + clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), + clusters_applications_helm: count(::Clusters::Applications::Helm.installed), + clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), + clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.installed), + clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), + clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + clusters_applications_knative: count(::Clusters::Applications::Knative.installed), + in_review_folder: count(::Environment.in_review_folder), + groups: count(Group), + issues: count(Issue), + keys: count(Key), + label_lists: count(List.label), + lfs_objects: count(LfsObject), + milestone_lists: count(List.milestone), + milestones: count(Milestone), + pages_domains: count(PagesDomain), + projects: count(Project), + projects_imported_from_github: count(Project.where(import_type: 'github')), + projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), + protected_branches: count(ProtectedBranch), + releases: count(Release), + remote_mirrors: count(RemoteMirror), + snippets: count(Snippet), + suggestions: count(Suggestion), + todos: count(Todo), + uploads: count(Upload), + web_hooks: count(WebHook) + }.merge(services_usage).merge(approximate_counts) } end + # rubocop: enable CodeReuse/ActiveRecord def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json @@ -90,13 +105,20 @@ module Gitlab def features_usage_data_ce { - signup: Gitlab::CurrentSettings.allow_signup?, - ldap: Gitlab.config.ldap.enabled, - gravatar: Gitlab::CurrentSettings.gravatar_enabled?, - omniauth: Gitlab.config.omniauth.enabled, - reply_by_email: Gitlab::IncomingEmail.enabled?, - container_registry: Gitlab.config.registry.enabled, - gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled + container_registry_enabled: Gitlab.config.registry.enabled, + gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, + gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, + ldap_enabled: Gitlab.config.ldap.enabled, + mattermost_enabled: Gitlab.config.mattermost.enabled, + omniauth_enabled: Gitlab::Auth.omniauth_enabled?, + reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, + signup_enabled: Gitlab::CurrentSettings.allow_signup? + } + end + + def usage_counters + { + web_ide_commits: Gitlab::WebIdeCommitsCounter.total_count } end @@ -108,16 +130,50 @@ module Gitlab } end + # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { - JiraService: :projects_jira_active, SlackService: :projects_slack_notifications_active, SlackSlashCommandsService: :projects_slack_slash_active, PrometheusService: :projects_prometheus_active } - results = Service.unscoped.where(type: types.keys, active: true).group(:type).count - results.each_with_object({}) { |(key, value), response| response[types[key.to_sym]] = value } + results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) + types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + .merge(jira_usage) + end + + def jira_usage + # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 + # so we can just check for subdomains of atlassian.net + services = count( + Service.unscoped.where(type: :JiraService, active: true) + .group("CASE WHEN properties LIKE '%.atlassian.net%' THEN 'cloud' ELSE 'server' END"), + fallback: Hash.new(-1) + ) + + { + projects_jira_server_active: services['server'] || 0, + projects_jira_cloud_active: services['cloud'] || 0, + projects_jira_active: services['server'] == -1 ? -1 : services.values.sum + } + end + + def count(relation, fallback: -1) + relation.count + rescue ActiveRecord::StatementInvalid + fallback + end + # rubocop: enable CodeReuse/ActiveRecord + + def approximate_counts + approx_counts = Gitlab::Database::Count.approximate_counts(APPROXIMATE_COUNT_MODELS) + + APPROXIMATE_COUNT_MODELS.each_with_object({}) do |model, result| + key = model.name.underscore.pluralize.to_sym + + result[key] = approx_counts[model] || -1 + end end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 8cf5d636743..980a8014409 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class UserAccess extend Gitlab::Cache::RequestCache @@ -65,7 +67,7 @@ module Gitlab return false unless can_access_git? return false unless project - return false if !user.can?(:push_code, project) && !project.branch_allows_maintainer_push?(user, ref) + return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref) if protected?(ProtectedBranch, project, ref) protected_branch_accessible_to?(ref, action: :push) diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb deleted file mode 100644 index 125488536e1..00000000000 --- a/lib/gitlab/user_activities.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - class UserActivities - include Enumerable - - KEY = 'users:activities'.freeze - BATCH_SIZE = 500 - - def self.record(key, time = Time.now) - Gitlab::Redis::SharedState.with do |redis| - redis.hset(KEY, key, time.to_i) - end - end - - def delete(*keys) - Gitlab::Redis::SharedState.with do |redis| - redis.hdel(KEY, keys) - end - end - - def each - cursor = 0 - loop do - cursor, pairs = - Gitlab::Redis::SharedState.with do |redis| - redis.hscan(KEY, cursor, count: BATCH_SIZE) - end - - Hash[pairs].each { |pair| yield pair } - - break if cursor == '0' - end - end - end -end diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb new file mode 100644 index 00000000000..874599688bb --- /dev/null +++ b/lib/gitlab/user_extractor.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# This class extracts all users found in a piece of text by the username or the +# email address + +module Gitlab + class UserExtractor + # Not using `Devise.email_regexp` to filter out any chars that an email + # does not end with and not pinning the email to a start of end of a string. + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + USERNAME_REGEXP = User.reference_pattern + + def initialize(text) + @text = text + end + + def users + return User.none unless @text.present? + + @users ||= User.from_union(union_relations) + end + + def usernames + matches[:usernames] + end + + def emails + matches[:emails] + end + + def references + @references ||= matches.values.flatten + end + + def matches + @matches ||= { + emails: @text.scan(EMAIL_REGEXP).flatten.uniq, + usernames: @text.scan(USERNAME_REGEXP).flatten.uniq + } + end + + private + + def union_relations + relations = [] + + relations << User.by_any_email(emails) if emails.any? + relations << User.by_username(usernames) if usernames.any? + + relations + end + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index aeda66763e8..99fa65e0e90 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -1,7 +1,18 @@ +# frozen_string_literal: true + module Gitlab module Utils extend self + # Ensure that the relative path will not traverse outside the base directory + def check_path_traversal!(path) + raise StandardError.new("Invalid path") if path.start_with?("..#{File::SEPARATOR}") || + path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") || + path.end_with?("#{File::SEPARATOR}..") + + path + end + # Run system command without outputting to stdout. # # @param cmd [Array<String>] @@ -14,6 +25,26 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + def ensure_utf8_size(str, bytes:) + raise ArgumentError, 'Empty string provided!' if str.empty? + raise ArgumentError, 'Negative string size provided!' if bytes.negative? + + truncated = str.each_char.each_with_object(+'') do |char, object| + if object.bytesize + char.bytesize > bytes + break object + else + object.concat(char) + end + end + + truncated + ('0' * (bytes - truncated.bytesize)) + end + + # Append path to host, making sure there's one single / in between + def append_path(host, path) + "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + end + # A slugified version of the string, suitable for inclusion in URLs and # domain names. Rules: # @@ -29,7 +60,7 @@ module Gitlab # Converts newlines into HTML line break elements def nlbr(str) - ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe + ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe end def remove_line_breaks(str) @@ -84,5 +115,15 @@ module Gitlab string_or_array.split(',').map(&:strip) end + + def deep_indifferent_access(data) + if data.is_a?(Array) + data.map(&method(:deep_indifferent_access)) + elsif data.is_a?(Hash) + data.with_indifferent_access + else + data + end + end end end diff --git a/lib/gitlab/utils/merge_hash.rb b/lib/gitlab/utils/merge_hash.rb index 385141d44d0..fc237861e2f 100644 --- a/lib/gitlab/utils/merge_hash.rb +++ b/lib/gitlab/utils/merge_hash.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Utils module MergeHash diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 8bf6bcb1fe2..c87e97d0213 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,17 +1,14 @@ +# frozen_string_literal: true + module Gitlab module Utils module Override class Extension - def self.verify_class!(klass, method_name) - instance_method_defined?(klass, method_name) || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) - end - - def self.instance_method_defined?(klass, name, include_super: true) - klass.instance_methods(include_super).include?(name) || - klass.private_instance_methods(include_super).include?(name) + def self.verify_class!(klass, method_name, arity) + extension = new(klass) + parents = extension.parents_for(klass) + extension.verify_method!( + klass: klass, parents: parents, method_name: method_name, sub_method_arity: arity) end attr_reader :subject @@ -20,35 +17,77 @@ module Gitlab @subject = subject end - def add_method_name(method_name) - method_names << method_name - end - - def add_class(klass) - classes << klass + def parents_for(klass) + index = klass.ancestors.index(subject) + klass.ancestors.drop(index + 1) end def verify! classes.each do |klass| - index = klass.ancestors.index(subject) - parents = klass.ancestors.drop(index + 1) - - method_names.each do |method_name| - parents.any? do |parent| - self.class.instance_method_defined?( - parent, method_name, include_super: false) - end || - raise( - NotImplementedError.new( - "#{klass}\##{method_name} doesn't exist!")) + parents = parents_for(klass) + + method_names.each_pair do |method_name, arity| + verify_method!( + klass: klass, + parents: parents, + method_name: method_name, + sub_method_arity: arity) end end end + def verify_method!(klass:, parents:, method_name:, sub_method_arity:) + overridden_parent = parents.find do |parent| + instance_method_defined?(parent, method_name) + end + + raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent + + super_method_arity = find_direct_method(overridden_parent, method_name).arity + + unless arity_compatible?(sub_method_arity, super_method_arity) + raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}") + end + end + + def add_method_name(method_name, arity = nil) + method_names[method_name] = arity + end + + def add_class(klass) + classes << klass + end + + def verify_override?(method_name) + method_names.has_key?(method_name) + end + private + def instance_method_defined?(klass, name) + klass.instance_methods(false).include?(name) || + klass.private_instance_methods(false).include?(name) + end + + def find_direct_method(klass, name) + method = klass.instance_method(name) + method = method.super_method until method && klass == method.owner + method + end + + def arity_compatible?(sub_method_arity, super_method_arity) + if sub_method_arity >= 0 && super_method_arity >= 0 + # Regular arguments + sub_method_arity == super_method_arity + else + # It's too complex to check this case, just allow sub-method having negative arity + # But we don't allow sub_method_arity > 0 yet super_method_arity < 0 + sub_method_arity < 0 + end + end + def method_names - @method_names ||= [] + @method_names ||= {} end def classes @@ -78,27 +117,51 @@ module Gitlab def override(method_name) return unless ENV['STATIC_VERIFICATION'] + Override.extensions[self] ||= Extension.new(self) + Override.extensions[self].add_method_name(method_name) + end + + def method_added(method_name) + super + + return unless ENV['STATIC_VERIFICATION'] + return unless Override.extensions[self]&.verify_override?(method_name) + + method_arity = instance_method(method_name).arity if is_a?(Class) - Extension.verify_class!(self, method_name) + Extension.verify_class!(self, method_name, method_arity) else # We delay the check for modules - Override.extensions[self] ||= Extension.new(self) - Override.extensions[self].add_method_name(method_name) + Override.extensions[self].add_method_name(method_name, method_arity) end end def included(base = nil) - return super if base.nil? # Rails concern, ignoring it + super + queue_verification(base) if base + end + + def prepended(base = nil) + super + + queue_verification(base) if base + end + + def extended(mod = nil) super + queue_verification(mod.singleton_class) if mod + end + + def queue_verification(base) + return unless ENV['STATIC_VERIFICATION'] + if base.is_a?(Class) # We could check for Class in `override` # This could be `nil` if `override` was never called Override.extensions[self]&.add_class(base) end end - alias_method :prepended, :included - def self.extensions @extensions ||= {} end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index fe091f4611b..aa1f8e2fdda 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Utils module StrongMemoize diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index 1ef369a4b67..dbda19a4a66 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Verify class BatchVerifier @@ -7,13 +9,15 @@ module Gitlab @batch_size = batch_size @start = start @finish = finish + + fix_google_api_logger end # Yields a Range of IDs and a Hash of failed verifications (object => error) def run_batches(&blk) - relation.in_batches(of: batch_size, start: start, finish: finish) do |relation| # rubocop: disable Cop/InBatches - range = relation.first.id..relation.last.id - failures = run_batch(relation) + all_relation.in_batches(of: batch_size, start: start, finish: finish) do |batch| # rubocop: disable Cop/InBatches + range = batch.first.id..batch.last.id + failures = run_batch_for(batch) yield(range, failures) end @@ -29,24 +33,56 @@ module Gitlab private - def run_batch(relation) - relation.map { |upload| verify(upload) }.compact.to_h + def run_batch_for(batch) + batch.map { |upload| verify(upload) }.compact.to_h end def verify(object) + local?(object) ? verify_local(object) : verify_remote(object) + rescue => err + failure(object, err.inspect) + end + + def verify_local(object) expected = expected_checksum(object) actual = actual_checksum(object) - raise 'Checksum missing' unless expected.present? - raise 'Checksum mismatch' unless expected == actual + return failure(object, 'Checksum missing') unless expected.present? + return failure(object, 'Checksum mismatch') unless expected == actual + success + end + + # We don't calculate checksum for remote objects, so just check existence + def verify_remote(object) + return failure(object, 'Remote object does not exist') unless remote_object_exists?(object) + + success + end + + def success nil - rescue => err - [object, err] + end + + def failure(object, message) + [object, message] + end + + # It's already set to Logger::INFO, but acts as if it is set to + # Logger::DEBUG, and this fixes it... + def fix_google_api_logger + if Object.const_defined?('Google::Apis') + Google::Apis.logger.level = Logger::INFO + end end # This should return an ActiveRecord::Relation suitable for calling #in_batches on - def relation + def all_relation + raise NotImplementedError.new + end + + # Should return true if the object is stored locally + def local?(_object) raise NotImplementedError.new end @@ -59,6 +95,11 @@ module Gitlab def actual_checksum(_object) raise NotImplementedError.new end + + # Be sure to perform a hard check of the remote object (don't just check DB value) + def remote_object_exists?(object) + raise NotImplementedError.new + end end end end diff --git a/lib/gitlab/verify/job_artifacts.rb b/lib/gitlab/verify/job_artifacts.rb index 03500a61074..3b90c8b1a8e 100644 --- a/lib/gitlab/verify/job_artifacts.rb +++ b/lib/gitlab/verify/job_artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Verify class JobArtifacts < BatchVerifier @@ -11,10 +13,14 @@ module Gitlab private - def relation + def all_relation ::Ci::JobArtifact.all end + def local?(artifact) + artifact.local_store? + end + def expected_checksum(artifact) artifact.file_sha256 end @@ -22,6 +28,10 @@ module Gitlab def actual_checksum(artifact) Digest::SHA256.file(artifact.file.path).hexdigest end + + def remote_object_exists?(artifact) + artifact.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb index 970e2a7b718..20dbb7addff 100644 --- a/lib/gitlab/verify/lfs_objects.rb +++ b/lib/gitlab/verify/lfs_objects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Verify class LfsObjects < BatchVerifier @@ -11,8 +13,12 @@ module Gitlab private - def relation - LfsObject.with_files_stored_locally + def all_relation + LfsObject.all + end + + def local?(lfs_object) + lfs_object.local_store? end def expected_checksum(lfs_object) @@ -22,6 +28,10 @@ module Gitlab def actual_checksum(lfs_object) LfsObject.calculate_oid(lfs_object.file.path) end + + def remote_object_exists?(lfs_object) + lfs_object.file.file.exists? + end end end end diff --git a/lib/gitlab/verify/rake_task.rb b/lib/gitlab/verify/rake_task.rb index dd138e6b92b..3efed311237 100644 --- a/lib/gitlab/verify/rake_task.rb +++ b/lib/gitlab/verify/rake_task.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Verify class RakeTask @@ -45,7 +47,7 @@ module Gitlab return unless verbose? failures.each do |object, error| - say " - #{verifier.describe(object)}: #{error.inspect}".color(:red) + say " - #{verifier.describe(object)}: #{error}".color(:red) end end end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 0ffa71a6d72..875e8a120e9 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Verify class Uploads < BatchVerifier @@ -11,8 +13,14 @@ module Gitlab private - def relation - Upload.with_files_stored_locally + # rubocop: disable CodeReuse/ActiveRecord + def all_relation + Upload.all.preload(:model) + end + # rubocop: enable CodeReuse/ActiveRecord + + def local?(upload) + upload.local? end def expected_checksum(upload) @@ -22,6 +30,10 @@ module Gitlab def actual_checksum(upload) Upload.hexdigest(upload.absolute_path) end + + def remote_object_exists?(upload) + upload.build_uploader.file.exists? + end end end end diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index 6ee41e85cc9..aa6d5310161 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class VersionInfo include Comparable diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 36162faa1eb..5e70afe730a 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module View module Presenter @@ -11,8 +13,8 @@ module Gitlab attr_reader :subject - def can?(user, action, overriden_subject = nil) - super(user, action, overriden_subject || subject) + def can?(user, action, overridden_subject = nil) + super(user, action, overridden_subject || subject) end # delegate all #can? queries to the subject diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index 387ff0f5d43..4a90ab758fb 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module View module Presenter diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb index 570f0723e39..302697ff8eb 100644 --- a/lib/gitlab/view/presenter/factory.rb +++ b/lib/gitlab/view/presenter/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module View module Presenter diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb index b7653a0f3cc..31dcd1d4c4c 100644 --- a/lib/gitlab/view/presenter/simple.rb +++ b/lib/gitlab/view/presenter/simple.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module View module Presenter diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2612208a927..a3c7de87765 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Gitlab::VisibilityLevel module # # Define allowed public modes that can be used for diff --git a/lib/gitlab/web_ide_commits_counter.rb b/lib/gitlab/web_ide_commits_counter.rb new file mode 100644 index 00000000000..1cd9b5295b9 --- /dev/null +++ b/lib/gitlab/web_ide_commits_counter.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module WebIdeCommitsCounter + WEB_IDE_COMMITS_KEY = "WEB_IDE_COMMITS_COUNT".freeze + + class << self + def increment + Gitlab::Redis::SharedState.with { |redis| redis.incr(WEB_IDE_COMMITS_KEY) } + end + + def total_count + Gitlab::Redis::SharedState.with { |redis| redis.get(WEB_IDE_COMMITS_KEY).to_i } + end + end + end +end diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb index 529f7d6a8d6..fda41da5a94 100644 --- a/lib/gitlab/webpack/dev_server_middleware.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This Rack middleware is intended to proxy the webpack assets directory to the # webpack-dev-server. It is only intended for use in development. diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 0c343e5bc1d..1d2aff5e5b4 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'webpack/rails/manifest' module Gitlab diff --git a/lib/gitlab/wiki_file_finder.rb b/lib/gitlab/wiki_file_finder.rb new file mode 100644 index 00000000000..5303b3582ab --- /dev/null +++ b/lib/gitlab/wiki_file_finder.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + class WikiFileFinder < FileFinder + BATCH_SIZE = 100 + + attr_reader :repository + + def initialize(project, ref) + @project = project + @ref = ref + @repository = project.wiki.repository + end + + private + + def search_filenames(query) + safe_query = Regexp.escape(query.tr(' ', '-')) + safe_query = Regexp.new(safe_query, Regexp::IGNORECASE) + filenames = repository.ls_files(ref) + + filenames.grep(safe_query).first(BATCH_SIZE) + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e893e46ee86..265f6213a99 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'base64' require 'json' require 'securerandom' @@ -11,6 +13,7 @@ module Gitlab INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze + DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type'.freeze # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 # bytes https://tools.ietf.org/html/rfc4868#section-2.6 @@ -22,36 +25,37 @@ module Gitlab project = repository.project - { + attrs = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, - RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitConfigOptions: [], GitalyServer: { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) } } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end + + attrs end def send_git_blob(repository, blob) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'GetBlobRequest' => { - repository: repository.gitaly_repository.to_h, - oid: blob.id, - limit: -1 - } - } - else - { - 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id - } - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'GetBlobRequest' => { + repository: repository.gitaly_repository.to_h, + oid: blob.id, + limit: -1 + } + } [ SEND_DATA_HEADER, @@ -61,7 +65,7 @@ module Gitlab def send_git_archive(repository, ref:, format:, append_sha:) format ||= 'tar.gz' - format.downcase! + format = format.downcase params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? @@ -91,16 +95,12 @@ module Gitlab end def send_git_diff(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'RawDiffRequest' => Gitaly::RawDiffRequest.new( - gitaly_diff_or_patch_hash(repository, diff_refs) - ).to_json - } - else - workhorse_diff_or_patch_hash(repository, diff_refs) - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawDiffRequest' => Gitaly::RawDiffRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } [ SEND_DATA_HEADER, @@ -109,16 +109,12 @@ module Gitlab end def send_git_patch(repository, diff_refs) - params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) - { - 'GitalyServer' => gitaly_server_hash(repository), - 'RawPatchRequest' => Gitaly::RawPatchRequest.new( - gitaly_diff_or_patch_hash(repository, diff_refs) - ).to_json - } - else - workhorse_diff_or_patch_hash(repository, diff_refs) - end + params = { + 'GitalyServer' => gitaly_server_hash(repository), + 'RawPatchRequest' => Gitaly::RawPatchRequest.new( + gitaly_diff_or_patch_hash(repository, diff_refs) + ).to_json + } [ SEND_DATA_HEADER, @@ -231,14 +227,6 @@ module Gitlab } end - def workhorse_diff_or_patch_hash(repository, diff_refs) - { - 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.base_sha, - 'ShaTo' => diff_refs.head_sha - } - end - def gitaly_diff_or_patch_hash(repository, diff_refs) { repository: repository.gitaly_repository, diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb index 1aeaa387a49..56f056fd869 100644 --- a/lib/google_api/auth.rb +++ b/lib/google_api/auth.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module GoogleApi class Auth attr_reader :access_token, :redirect_uri, :state @@ -14,7 +16,7 @@ module GoogleApi client.auth_code.authorize_url( redirect_uri: redirect_uri, scope: scope, - state: state # This is used for arbitary redirection + state: state # This is used for arbitrary redirection ) end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 36859b4d025..e74ff6a9129 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'google/apis/compute_v1' require 'google/apis/container_v1' require 'google/apis/cloudbilling_v1' @@ -50,7 +52,7 @@ module GoogleApi service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) end - def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:) + def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token @@ -63,7 +65,7 @@ module GoogleApi "machine_type": machine_type }, "legacy_abac": { - "enabled": true + "enabled": legacy_abac } } } diff --git a/lib/gt_one_coercion.rb b/lib/gt_one_coercion.rb index ef2dc09767c..99be51bc8c6 100644 --- a/lib/gt_one_coercion.rb +++ b/lib/gt_one_coercion.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class GtOneCoercion < Virtus::Attribute def coerce(value) [1, value.to_i].max diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index adbed20f152..2e98227a05e 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -1,7 +1,10 @@ -unless Rails.env.production? # rubocop:disable Naming/FileName - require 'haml_lint/haml_visitor' - require 'haml_lint/linter' - require 'haml_lint/linter_registry' +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +unless Rails.env.production? + require_dependency 'haml_lint/haml_visitor' + require_dependency 'haml_lint/linter' + require_dependency 'haml_lint/linter_registry' module HamlLint class Linter::InlineJavaScript < Linter diff --git a/lib/json_web_token/hmac_token.rb b/lib/json_web_token/hmac_token.rb new file mode 100644 index 00000000000..ec0917ab49d --- /dev/null +++ b/lib/json_web_token/hmac_token.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'jwt' + +module JSONWebToken + class HMACToken < Token + IAT_LEEWAY = 60 + JWT_ALGORITHM = 'HS256' + + def initialize(secret) + super() + + @secret = secret + end + + def self.decode(token, secret, leeway: IAT_LEEWAY, verify_iat: true) + JWT.decode(token, secret, true, leeway: leeway, verify_iat: verify_iat, algorithm: JWT_ALGORITHM) + end + + def encoded + JWT.encode(payload, secret, JWT_ALGORITHM, { typ: 'JWT' }) + end + + private + + attr_reader :secret + end +end diff --git a/lib/json_web_token/rsa_token.rb b/lib/json_web_token/rsa_token.rb index d6d6af7089c..bcce811cd28 100644 --- a/lib/json_web_token/rsa_token.rb +++ b/lib/json_web_token/rsa_token.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module JSONWebToken class RSAToken < Token attr_reader :key_file @@ -9,7 +11,8 @@ module JSONWebToken def encoded headers = { - kid: kid + kid: kid, + typ: 'JWT' } JWT.encode(payload, key, 'RS256', headers) end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb index 5b67715b0b2..c59beef02c9 100644 --- a/lib/json_web_token/token.rb +++ b/lib/json_web_token/token.rb @@ -1,15 +1,22 @@ +# frozen_string_literal: true + +require 'securerandom' + module JSONWebToken class Token attr_accessor :issuer, :subject, :audience, :id attr_accessor :issued_at, :not_before, :expire_time + DEFAULT_NOT_BEFORE_TIME = 5 + DEFAULT_EXPIRE_TIME = 60 + def initialize @id = SecureRandom.uuid @issued_at = Time.now # we give a few seconds for time shift - @not_before = issued_at - 5.seconds + @not_before = issued_at - DEFAULT_NOT_BEFORE_TIME # default 60 seconds should be more than enough for this authentication token - @expire_time = issued_at + 1.minute + @expire_time = issued_at + DEFAULT_EXPIRE_TIME @custom_payload = {} end diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index d80cd7d2a4e..293d0c563c5 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Mattermost ClientError = Class.new(Mattermost::Error) diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb index 704813dfdf0..a02745486d6 100644 --- a/lib/mattermost/command.rb +++ b/lib/mattermost/command.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Mattermost class Command < Client def create(params) diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb index dee6deb7974..054bd5457bd 100644 --- a/lib/mattermost/error.rb +++ b/lib/mattermost/error.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Mattermost Error = Class.new(StandardError) end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 2aa7a2f64d8..e2083848a8d 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Mattermost class NoSessionError < Mattermost::Error def message diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 95c2f6f9d6b..58120178f50 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Mattermost class Team < Client # Returns all teams that the current user is a member of diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb index d2c420efdaf..207e90d2638 100644 --- a/lib/microsoft_teams/activity.rb +++ b/lib/microsoft_teams/activity.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module MicrosoftTeams class Activity def initialize(title:, subtitle:, text:, image:) diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index c08d3e933a8..c7dec09ba6b 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module MicrosoftTeams class Notifier def initialize(webhook) @@ -30,7 +32,7 @@ module MicrosoftTeams result = { 'sections' => [] } result['title'] = options[:title] - result['summary'] = options[:pretext] + result['summary'] = options[:summary] result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare attachments = options[:attachments] diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb index 4ed8485b36a..461e73e9670 100644 --- a/lib/milestone_array.rb +++ b/lib/milestone_array.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module MilestoneArray class << self def sort(array, sort_method) diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb new file mode 100644 index 00000000000..f36610abf8f --- /dev/null +++ b/lib/mysql_zero_date.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Disable NO_ZERO_DATE mode for mysql in rails 5. +# We use zero date as a default value +# (config/initializers/active_record_mysql_timestamp.rb), in +# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) +# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode +# is not sufficient. + +require 'active_record/connection_adapters/abstract_mysql_adapter' + +module MysqlZeroDate + def configure_connection + super + + @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end + +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb new file mode 100644 index 00000000000..fd26663fef0 --- /dev/null +++ b/lib/object_storage/direct_upload.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +module ObjectStorage + # + # The DirectUpload c;ass generates a set of presigned URLs + # that can be used to upload data to object storage from untrusted component: Workhorse, Runner? + # + # For Google it assumes that the platform supports variable Content-Length. + # + # For AWS it initiates Multipart Upload and presignes a set of part uploads. + # Class calculates the best part size to be able to upload up to asked maximum size. + # The number of generated parts will never go above 100, + # but we will always try to reduce amount of generated parts. + # The part size is rounded-up to 5MB. + # + class DirectUpload + include Gitlab::Utils::StrongMemoize + + TIMEOUT = 4.hours + EXPIRE_OFFSET = 15.minutes + + MAXIMUM_MULTIPART_PARTS = 100 + MINIMUM_MULTIPART_SIZE = 5.megabytes + + attr_reader :credentials, :bucket_name, :object_name + attr_reader :has_length, :maximum_size + + def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil) + unless has_length + raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size + end + + @credentials = credentials + @bucket_name = bucket_name + @object_name = object_name + @has_length = has_length + @maximum_size = maximum_size + end + + def to_hash + { + Timeout: TIMEOUT, + GetURL: get_url, + StoreURL: store_url, + DeleteURL: delete_url, + MultipartUpload: multipart_upload_hash, + CustomPutHeaders: true, + PutHeaders: upload_options + }.compact + end + + def multipart_upload_hash + return unless requires_multipart_upload? + + { + PartSize: rounded_multipart_part_size, + PartURLs: multipart_part_urls, + CompleteURL: multipart_complete_url, + AbortURL: multipart_abort_url + } + end + + def provider + credentials[:provider].to_s + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + def get_url + connection.get_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html + def delete_url + connection.delete_object_url(bucket_name, object_name, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html + def store_url + connection.put_object_url(bucket_name, object_name, expire_at, upload_options) + end + + def multipart_part_urls + Array.new(number_of_multipart_parts) do |part_index| + multipart_part_upload_url(part_index + 1) + end + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + def multipart_part_upload_url(part_number) + connection.signed_url({ + method: 'PUT', + bucket_name: bucket_name, + object_name: object_name, + query: { 'uploadId' => upload_id, 'partNumber' => part_number }, + headers: upload_options + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html + def multipart_complete_url + connection.signed_url({ + method: 'POST', + bucket_name: bucket_name, + object_name: object_name, + query: { 'uploadId' => upload_id }, + headers: { 'Content-Type' => 'application/xml' } + }, expire_at) + end + + # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html + def multipart_abort_url + connection.signed_url({ + method: 'DELETE', + bucket_name: bucket_name, + object_name: object_name, + query: { 'uploadId' => upload_id } + }, expire_at) + end + + private + + def rounded_multipart_part_size + # round multipart_part_size up to minimum_mulitpart_size + (multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE + end + + def multipart_part_size + maximum_size / number_of_multipart_parts + end + + def number_of_multipart_parts + [ + # round maximum_size up to minimum_mulitpart_size + (maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE, + MAXIMUM_MULTIPART_PARTS + ].min + end + + def aws? + provider == 'AWS' + end + + def requires_multipart_upload? + aws? && !has_length + end + + def upload_id + return unless requires_multipart_upload? + + strong_memoize(:upload_id) do + new_upload = connection.initiate_multipart_upload(bucket_name, object_name) + new_upload.body["UploadId"] + end + end + + def expire_at + strong_memoize(:expire_at) do + Time.now + TIMEOUT + EXPIRE_OFFSET + end + end + + def upload_options + {} + end + + def connection + @connection ||= ::Fog::Storage.new(credentials) + end + end +end diff --git a/lib/omni_auth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb index ce1bdfe6ee4..6c914b4222a 100644 --- a/lib/omni_auth/strategies/bitbucket.rb +++ b/lib/omni_auth/strategies/bitbucket.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'omniauth-oauth2' module OmniAuth diff --git a/lib/omni_auth/strategies/jwt.rb b/lib/omni_auth/strategies/jwt.rb index ebdb5c7faf0..2f3d477a591 100644 --- a/lib/omni_auth/strategies/jwt.rb +++ b/lib/omni_auth/strategies/jwt.rb @@ -1,4 +1,7 @@ +# frozen_string_literal: true + require 'omniauth' +require 'openssl' require 'jwt' module OmniAuth @@ -35,7 +38,19 @@ module OmniAuth end def decoded - @decoded ||= ::JWT.decode(request.params['jwt'], options.secret, options.algorithm).first + secret = + case options.algorithm + when *%w[RS256 RS384 RS512] + OpenSSL::PKey::RSA.new(options.secret).public_key + when *%w[ES256 ES384 ES512] + OpenSSL::PKey::EC.new(options.secret).tap { |key| key.private_key = nil } + when *%w(HS256 HS384 HS512) + options.secret + else + raise NotImplementedError, "Unsupported algorithm: #{options.algorithm}" + end + + @decoded ||= ::JWT.decode(request.params['jwt'], secret, true, { algorithm: options.algorithm }).first (options.required_claims || []).each do |field| raise ClaimInvalid, "Missing required '#{field}' claim" unless @decoded.key?(field.to_s) @@ -43,7 +58,7 @@ module OmniAuth raise ClaimInvalid, "Missing required 'iat' claim" if options.valid_within && !@decoded["iat"] - if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within + if options.valid_within && (Time.now.to_i - @decoded["iat"]).abs > options.valid_within.to_i raise ClaimInvalid, "'iat' timestamp claim is too skewed from present" end diff --git a/lib/peek/rblineprof/custom_controller_helpers.rb b/lib/peek/rblineprof/custom_controller_helpers.rb index da24a36603e..581cc6a37b4 100644 --- a/lib/peek/rblineprof/custom_controller_helpers.rb +++ b/lib/peek/rblineprof/custom_controller_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Peek module Rblineprof module CustomControllerHelpers @@ -41,7 +43,7 @@ module Peek ] end.sort_by{ |a,b,c,d,e,f| -f } - output = "<div class='modal-dialog modal-lg'><div class='modal-content'>" + output = ["<div class='modal-dialog modal-xl'><div class='modal-content'>"] output << "<div class='modal-header'>" output << "<h4>Line profiling: #{human_description(params[:lineprofiler])}</h4>" output << "<button class='close' type='button' data-dismiss='modal' aria-label='close'><span aria-hidden='true'>×</span></button>" @@ -93,7 +95,7 @@ module Peek output << "</div></div></div>" - response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output}</div>".html_safe + response.body += "<div class='modal' id='modal-peek-line-profile' tabindex=-1>#{output.join}</div>".html_safe end ret diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb index ab35f7a2258..30f95a10024 100644 --- a/lib/peek/views/gitaly.rb +++ b/lib/peek/views/gitaly.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Peek module Views class Gitaly < View @@ -21,7 +23,6 @@ module Peek def details ::Gitlab::GitalyClient.list_call_details - .values .sort { |a, b| b[:duration] <=> a[:duration] } .map(&method(:format_call_details)) end diff --git a/lib/peek/views/host.rb b/lib/peek/views/host.rb index 43c8a35c7ea..b77355ea11b 100644 --- a/lib/peek/views/host.rb +++ b/lib/peek/views/host.rb @@ -1,8 +1,13 @@ +# frozen_string_literal: true + module Peek module Views class Host < View def results - { hostname: Gitlab::Environment.hostname } + { + hostname: Gitlab::Environment.hostname, + canary: Gitlab::Utils.to_boolean(ENV['CANARY']) + } end end end diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb new file mode 100644 index 00000000000..cf1f03b35b5 --- /dev/null +++ b/lib/quality/helm_client.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'time' +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class HelmClient + CommandFailedError = Class.new(StandardError) + + attr_reader :namespace + + RELEASE_JSON_ATTRIBUTES = %w[Name Revision Updated Status Chart AppVersion Namespace].freeze + + Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do + def revision + @revision ||= self[:revision].to_i + end + + def last_update + @last_update ||= Time.parse(self[:last_update]) + end + end + + # A single page of data and the corresponding page number. + Page = Struct.new(:releases, :number) + + def initialize(namespace:) + @namespace = namespace + end + + def releases(args: []) + each_release(args) + end + + def delete(release_name:) + run_command([ + 'delete', + %(--tiller-namespace "#{namespace}"), + '--purge', + release_name + ]) + end + + private + + def run_command(command) + final_command = ['helm', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + result = Gitlab::Popen.popen_with_detail([final_command]) + + if result.status.success? + result.stdout.chomp.freeze + else + raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + end + + def raw_releases(args = []) + command = [ + 'list', + %(--namespace "#{namespace}"), + %(--tiller-namespace "#{namespace}" --output json), + *args + ] + json = JSON.parse(run_command(command)) + + releases = json['Releases'].map do |json_release| + Release.new(*json_release.values_at(*RELEASE_JSON_ATTRIBUTES)) + end + + [releases, json['Next']] + rescue JSON::ParserError => ex + puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output + [[], nil] + end + + # Fetches data from Helm and yields a Page object for every page + # of data, without loading all of them into memory. + # + # method - The Octokit method to use for getting the data. + # args - Arguments to pass to the `helm list` command. + def each_releases_page(args, &block) + return to_enum(__method__, args) unless block_given? + + page = 1 + offset = '' + + loop do + final_args = args.dup + final_args << "--offset #{offset}" unless offset.to_s.empty? + collection, offset = raw_releases(final_args) + + yield Page.new(collection, page += 1) + + break if offset.to_s.empty? + end + end + + # Iterates over all of the releases. + # + # args - Any arguments to pass to the `helm list` command. + def each_release(args, &block) + return to_enum(__method__, args) unless block_given? + + each_releases_page(args) do |page| + page.releases.each do |release| + yield release + end + end + end + end +end diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb new file mode 100644 index 00000000000..2ff9e811425 --- /dev/null +++ b/lib/quality/kubernetes_client.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require_relative '../gitlab/popen' unless defined?(Gitlab::Popen) + +module Quality + class KubernetesClient + CommandFailedError = Class.new(StandardError) + + attr_reader :namespace + + def initialize(namespace:) + @namespace = namespace + end + + def cleanup(release_name:) + command = [ + %(--namespace "#{namespace}"), + 'delete', + 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa', + '--now', + %(-l release="#{release_name}") + ] + + run_command(command) + end + + private + + def run_command(command) + final_command = ['kubectl', *command].join(' ') + puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + + result = Gitlab::Popen.popen_with_detail([final_command]) + + if result.status.success? + result.stdout.chomp.freeze + else + raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + end + end + end +end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index be0d97370d0..e2a7d3ef5ba 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Rouge module Formatters class HTMLGitlab < Rouge::Formatters::HTML @@ -11,7 +13,7 @@ module Rouge @tag = tag end - def stream(tokens, &b) + def stream(tokens) is_first = true token_lines(tokens) do |line| yield "\n" unless is_first diff --git a/lib/rouge/plugins/common_mark.rb b/lib/rouge/plugins/common_mark.rb index 8f9de061124..d240df5a0e0 100644 --- a/lib/rouge/plugins/common_mark.rb +++ b/lib/rouge/plugins/common_mark.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # A rouge plugin for CommonMark markdown engine. # Used to highlight code generated by CommonMark. diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb index 06e96f969f1..55c1d4747b4 100644 --- a/lib/rspec_flaky/config.rb +++ b/lib/rspec_flaky/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module RspecFlaky class Config def self.generate_report? diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb index b6e790cbbab..3c1b05257a0 100644 --- a/lib/rspec_flaky/example.rb +++ b/lib/rspec_flaky/example.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module RspecFlaky # This is a wrapper class for RSpec::Core::Example class Example diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb index 6be24014d89..da5dbf06bc9 100644 --- a/lib/rspec_flaky/flaky_example.rb +++ b/lib/rspec_flaky/flaky_example.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module RspecFlaky # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file class FlakyExample < OpenStruct diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb index dea23c325be..290a51766e9 100644 --- a/lib/rspec_flaky/flaky_examples_collection.rb +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'active_support/hash_with_indifferent_access' require_relative 'flaky_example' diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index 9cd0c38cb55..19cc0baa2d3 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'json' require_dependency 'rspec_flaky/config' diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb index 1c362fdd20d..9a0fb88c424 100644 --- a/lib/rspec_flaky/report.rb +++ b/lib/rspec_flaky/report.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'json' require 'time' diff --git a/lib/safe_zip/entry.rb b/lib/safe_zip/entry.rb new file mode 100644 index 00000000000..664e2f52f91 --- /dev/null +++ b/lib/safe_zip/entry.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module SafeZip + class Entry + attr_reader :zip_archive, :zip_entry + attr_reader :path, :params + + def initialize(zip_archive, zip_entry, params) + @zip_archive = zip_archive + @zip_entry = zip_entry + @params = params + @path = ::File.expand_path(zip_entry.name, params.extract_path) + end + + def path_dir + ::File.dirname(path) + end + + def real_path_dir + ::File.realpath(path_dir) + end + + def exist? + ::File.exist?(path) + end + + def extract + # do not extract if file is not part of target directory + return false unless matching_target_directory + + # do not overwrite existing file + raise SafeZip::Extract::AlreadyExistsError, "File already exists #{zip_entry.name}" if exist? + + create_path_dir + + if zip_entry.file? + extract_file + elsif zip_entry.directory? + extract_dir + elsif zip_entry.symlink? + extract_symlink + else + raise SafeZip::Extract::UnsupportedEntryError, "File #{zip_entry.name} cannot be extracted" + end + rescue SafeZip::Extract::Error + raise + rescue => e + raise SafeZip::Extract::ExtractError, e.message + end + + private + + def extract_file + zip_archive.extract(zip_entry, path) + end + + def extract_dir + FileUtils.mkdir(path) + end + + def extract_symlink + source_path = read_symlink + real_source_path = expand_symlink(source_path) + + # ensure that source path of symlink is within target directories + unless real_source_path.start_with?(matching_target_directory) + raise SafeZip::Extract::PermissionDeniedError, "Symlink cannot be created targeting: #{source_path}" + end + + ::File.symlink(source_path, path) + end + + def create_path_dir + # Create all directories, but ignore permissions + FileUtils.mkdir_p(path_dir) + + # disallow to make path dirs to point to another directories + unless path_dir == real_path_dir + raise SafeZip::Extract::PermissionDeniedError, "Directory of #{zip_entry.name} points to another directory" + end + end + + def matching_target_directory + params.matching_target_directory(path) + end + + def read_symlink + zip_archive.read(zip_entry) + end + + def expand_symlink(source_path) + ::File.realpath(source_path, path_dir) + rescue + raise SafeZip::Extract::SymlinkSourceDoesNotExistError, "Symlink source #{source_path} does not exist" + end + end +end diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb new file mode 100644 index 00000000000..679c021c730 --- /dev/null +++ b/lib/safe_zip/extract.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module SafeZip + class Extract + Error = Class.new(StandardError) + PermissionDeniedError = Class.new(Error) + SymlinkSourceDoesNotExistError = Class.new(Error) + UnsupportedEntryError = Class.new(Error) + AlreadyExistsError = Class.new(Error) + NoMatchingError = Class.new(Error) + ExtractError = Class.new(Error) + + attr_reader :archive_path + + def initialize(archive_file) + @archive_path = archive_file + end + + def extract(opts = {}) + params = SafeZip::ExtractParams.new(**opts) + + if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true) + extract_with_ruby_zip(params) + else + legacy_unsafe_extract_with_system_zip(params) + end + end + + private + + def extract_with_ruby_zip(params) + ::Zip::File.open(archive_path) do |zip_archive| + # Extract all files in the following order: + # 1. Directories first, + # 2. Files next, + # 3. Symlinks last (or anything else) + extracted = extract_all_entries(zip_archive, params, + zip_archive.lazy.select(&:directory?)) + + extracted += extract_all_entries(zip_archive, params, + zip_archive.lazy.select(&:file?)) + + extracted += extract_all_entries(zip_archive, params, + zip_archive.lazy.reject(&:directory?).reject(&:file?)) + + raise NoMatchingError, 'No entries extracted' unless extracted > 0 + end + end + + def extract_all_entries(zip_archive, params, entries) + entries.count do |zip_entry| + SafeZip::Entry.new(zip_archive, zip_entry, params) + .extract + end + end + + def legacy_unsafe_extract_with_system_zip(params) + # Requires UnZip at least 6.00 Info-ZIP. + # -n never overwrite existing files + args = %W(unzip -n -qq #{archive_path}) + + # We add * to end of directory, because we want to extract directory and all subdirectories + args += params.directories_wildcard + + # Target directory where we extract + args += %W(-d #{params.extract_path}) + + unless system(*args) + raise Error, 'archive failed to extract' + end + end + end +end diff --git a/lib/safe_zip/extract_params.rb b/lib/safe_zip/extract_params.rb new file mode 100644 index 00000000000..bd3b788bac9 --- /dev/null +++ b/lib/safe_zip/extract_params.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module SafeZip + class ExtractParams + include Gitlab::Utils::StrongMemoize + + attr_reader :directories, :extract_path + + def initialize(directories:, to:) + @directories = directories + @extract_path = ::File.realpath(to) + end + + def matching_target_directory(path) + target_directories.find do |directory| + path.start_with?(directory) + end + end + + def target_directories + strong_memoize(:target_directories) do + directories.map do |directory| + ::File.join(::File.expand_path(directory, extract_path), '') + end + end + end + + def directories_wildcard + strong_memoize(:directories_wildcard) do + directories.map do |directory| + ::File.join(directory, '*') + end + end + end + end +end diff --git a/lib/sentry/client.rb b/lib/sentry/client.rb new file mode 100644 index 00000000000..4187014d49e --- /dev/null +++ b/lib/sentry/client.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Sentry + class Client + Error = Class.new(StandardError) + SentryError = Class.new(StandardError) + + attr_accessor :url, :token + + def initialize(api_url, token) + @url = api_url + @token = token + end + + def list_issues(issue_status:, limit:) + issues = get_issues(issue_status: issue_status, limit: limit) + map_to_errors(issues) + end + + def list_projects + projects = get_projects + map_to_projects(projects) + rescue KeyError => e + raise Client::SentryError, "Sentry API response is missing keys. #{e.message}" + end + + private + + def request_params + { + headers: { + 'Authorization' => "Bearer #{@token}" + }, + follow_redirects: false + } + end + + def http_get(url, params = {}) + resp = Gitlab::HTTP.get(url, **request_params.merge(params)) + + handle_response(resp) + end + + def get_issues(issue_status:, limit:) + http_get(issues_api_url, query: { + query: "is:#{issue_status}", + limit: limit + }) + end + + def get_projects + http_get(projects_api_url) + end + + def handle_response(response) + unless response.code == 200 + raise Client::Error, "Sentry response error: #{response.code}" + end + + response.as_json + end + + def projects_api_url + projects_url = URI(@url) + projects_url.path = '/api/0/projects/' + + projects_url + end + + def issues_api_url + issues_url = URI(@url + '/issues/') + issues_url.path.squeeze!('/') + + issues_url + end + + def map_to_errors(issues) + issues.map(&method(:map_to_error)) + end + + def map_to_projects(projects) + projects.map(&method(:map_to_project)) + end + + def issue_url(id) + issues_url = @url + "/issues/#{id}" + issues_url = ErrorTracking::ProjectErrorTrackingSetting.extract_sentry_external_url(issues_url) + + uri = URI(issues_url) + uri.path.squeeze!('/') + + uri.to_s + end + + def map_to_error(issue) + id = issue.fetch('id') + project = issue.fetch('project') + + count = issue.fetch('count', nil) + + frequency = issue.dig('stats', '24h') + message = issue.dig('metadata', 'value') + + external_url = issue_url(id) + + Gitlab::ErrorTracking::Error.new( + id: id, + first_seen: issue.fetch('firstSeen', nil), + last_seen: issue.fetch('lastSeen', nil), + title: issue.fetch('title', nil), + type: issue.fetch('type', nil), + user_count: issue.fetch('userCount', nil), + count: count, + message: message, + culprit: issue.fetch('culprit', nil), + external_url: external_url, + short_id: issue.fetch('shortId', nil), + status: issue.fetch('status', nil), + frequency: frequency, + project_id: project.fetch('id'), + project_name: project.fetch('name', nil), + project_slug: project.fetch('slug', nil) + ) + end + + def map_to_project(project) + organization = project.fetch('organization') + + Gitlab::ErrorTracking::Project.new( + id: project.fetch('id'), + name: project.fetch('name'), + slug: project.fetch('slug'), + status: project.dig('status'), + organization_name: organization.fetch('name'), + organization_id: organization.fetch('id'), + organization_slug: organization.fetch('slug') + ) + end + end +end diff --git a/lib/serializers/json.rb b/lib/serializers/json.rb new file mode 100644 index 00000000000..93cb192087a --- /dev/null +++ b/lib/serializers/json.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Serializers + # This serializer exports data as JSON, + # it is designed to be used with interwork compatibility between MySQL and PostgreSQL + # implementations, as used version of MySQL does not support native json type + # + # Secondly, the loader makes the resulting hash to have deep indifferent access + class JSON + class << self + def dump(obj) + # MySQL stores data as text + # look at ./config/initializers/ar_mysql_jsonb_support.rb + if Gitlab::Database.mysql? + obj = ActiveSupport::JSON.encode(obj) + end + + obj + end + + def load(data) + return if data.nil? + + # On MySQL we store data as text + # look at ./config/initializers/ar_mysql_jsonb_support.rb + if Gitlab::Database.mysql? + data = ActiveSupport::JSON.decode(data) + end + + Gitlab::Utils.deep_indifferent_access(data) + end + end + end +end diff --git a/lib/static_model.rb b/lib/static_model.rb index 60e2dd82e4e..86bf8d62f9a 100644 --- a/lib/static_model.rb +++ b/lib/static_model.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + # Provides an ActiveRecord-like interface to a model whose data is not persisted to a database. module StaticModel extend ActiveSupport::Concern - module ClassMethods + class_methods do # Used by ActiveRecord's polymorphic association to set object_id def primary_key 'id' diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 72eb8adcce2..fc984d737d5 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -17,7 +17,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -112,7 +112,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public. root /home/git/gitlab/public; internal; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 2e3799d5e1b..ba01e250bbb 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -21,7 +21,7 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { - # Gitlab socket file, + # GitLab socket file, # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -162,7 +162,7 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { - # Location to the Gitlab's public directory, + # Location to the GitLab's public directory, # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public root /home/git/gitlab/public; internal; diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl index 92511e26861..908d26a0da2 100644 --- a/lib/support/nginx/registry-ssl +++ b/lib/support/nginx/registry-ssl @@ -10,7 +10,7 @@ server { listen *:80; server_name registry.gitlab.example.com; server_tokens off; ## Don't show the nginx version number, a security best practice - return 301 https://$http_host:$request_uri; + return 301 https://$http_host$request_uri; access_log /var/log/nginx/gitlab_registry_access.log gitlab_access; error_log /var/log/nginx/gitlab_registry_error.log; } diff --git a/lib/system_check.rb b/lib/system_check.rb index 466c39904fa..7ffd7c03c5b 100644 --- a/lib/system_check.rb +++ b/lib/system_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Library to perform System Checks # # Every Check is implemented as its own class inherited from SystemCheck::BaseCheck diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb index 1d72c8d6903..8446c2fc2c8 100644 --- a/lib/system_check/app/active_users_check.rb +++ b/lib/system_check/app/active_users_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class ActiveUsersCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb index d1fae192350..1769145ed63 100644 --- a/lib/system_check/app/database_config_exists_check.rb +++ b/lib/system_check/app/database_config_exists_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class DatabaseConfigExistsCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb index d08a81639e3..4e8d607096c 100644 --- a/lib/system_check/app/git_config_check.rb +++ b/lib/system_check/app/git_config_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class GitConfigCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index ad41760dff2..6cd53779bfd 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class GitUserDefaultSSHConfigCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 44ec888c197..7c3e7759dd0 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class GitVersionCheck < SystemCheck::BaseCheck @@ -5,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.new(2, 9, 5) + @required_version ||= Gitlab::VersionInfo.parse('2.18.0') end def self.current_version diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb index 247aa0994e4..1cc5ead0d89 100644 --- a/lib/system_check/app/gitlab_config_exists_check.rb +++ b/lib/system_check/app/gitlab_config_exists_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class GitlabConfigExistsCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb index c609e48e133..58c7e3039c8 100644 --- a/lib/system_check/app/gitlab_config_up_to_date_check.rb +++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb index d246e058e86..d36dbe7d67d 100644 --- a/lib/system_check/app/init_script_exists_check.rb +++ b/lib/system_check/app/init_script_exists_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class InitScriptExistsCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb index 53a47eb0f42..569c41df6e4 100644 --- a/lib/system_check/app/init_script_up_to_date_check.rb +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class InitScriptUpToDateCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb index 3e0c436d6ee..e26ad143eb8 100644 --- a/lib/system_check/app/log_writable_check.rb +++ b/lib/system_check/app/log_writable_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class LogWritableCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb index 5eedbacce77..b12e9ac6bba 100644 --- a/lib/system_check/app/migrations_are_up_check.rb +++ b/lib/system_check/app/migrations_are_up_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class MigrationsAreUpCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb index 2b46d36fe51..3e6ffb8190b 100644 --- a/lib/system_check/app/orphaned_group_members_check.rb +++ b/lib/system_check/app/orphaned_group_members_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class OrphanedGroupMembersCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb index a6ec9f7665c..2bf2529acf1 100644 --- a/lib/system_check/app/projects_have_namespace_check.rb +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb index a0610e73576..890f8b44d13 100644 --- a/lib/system_check/app/redis_version_check.rb +++ b/lib/system_check/app/redis_version_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class RedisVersionCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index 57bbabece1f..60e07718338 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class RubyVersionCheck < SystemCheck::BaseCheck @@ -9,7 +11,7 @@ module SystemCheck end def self.current_version - @current_version ||= Gitlab::VersionInfo.parse(Gitlab::TaskHelpers.run_command(%w(ruby --version))) + @current_version ||= Gitlab::VersionInfo.parse(RUBY_VERSION) end def check? diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb index 99a75e57abf..6687df091d3 100644 --- a/lib/system_check/app/tmp_writable_check.rb +++ b/lib/system_check/app/tmp_writable_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class TmpWritableCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb index 7026d0ba075..940eff9d4cf 100644 --- a/lib/system_check/app/uploads_directory_exists_check.rb +++ b/lib/system_check/app/uploads_directory_exists_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb index 7df6c060254..4a49f3bc2bb 100644 --- a/lib/system_check/app/uploads_path_permission_check.rb +++ b/lib/system_check/app/uploads_path_permission_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class UploadsPathPermissionCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb index b276a81eac1..ae374f4707c 100644 --- a/lib/system_check/app/uploads_path_tmp_permission_check.rb +++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module App class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index 0f5742dd67f..46aad8aa885 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck # Base class for Checks. You must inherit from here # and implement the methods below when necessary @@ -68,18 +70,14 @@ module SystemCheck # multiple reasons why a check can fail # # @param [String] reason to be displayed - def skip_reason=(reason) - @skip_reason = reason - end + attr_writer :skip_reason # Skip reason defined during runtime # # This value have precedence over the one defined in the subclass # # @return [String] the reason - def skip_reason - @skip_reason - end + attr_reader :skip_reason # Does the check support automatically repair routine? # diff --git a/lib/system_check/gitaly_check.rb b/lib/system_check/gitaly_check.rb new file mode 100644 index 00000000000..3d2517a7aca --- /dev/null +++ b/lib/system_check/gitaly_check.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module SystemCheck + class GitalyCheck < BaseCheck + set_name 'Gitaly:' + + def multi_check + Gitlab::HealthChecks::GitalyCheck.readiness.each do |result| + $stdout.print "#{result.labels[:shard]} ... " + + if result.success + $stdout.puts 'OK'.color(:green) + else + $stdout.puts "FAIL: #{result.message}".color(:red) + end + end + end + end +end diff --git a/lib/system_check/gitlab_shell_check.rb b/lib/system_check/gitlab_shell_check.rb new file mode 100644 index 00000000000..31c4ec33247 --- /dev/null +++ b/lib/system_check/gitlab_shell_check.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:gitlab_shell:check rake task + class GitlabShellCheck < BaseCheck + set_name 'GitLab Shell:' + + def multi_check + check_gitlab_shell + check_gitlab_shell_self_test + end + + private + + def check_gitlab_shell + required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required) + current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) + + $stdout.print "GitLab Shell version >= #{required_version} ? ... " + if current_version.valid? && required_version <= current_version + $stdout.puts "OK (#{current_version})".color(:green) + else + $stdout.puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) + end + end + + def check_gitlab_shell_self_test + gitlab_shell_repo_base = gitlab_shell_path + check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) + $stdout.puts "Running #{check_cmd}" + + if system(check_cmd, chdir: gitlab_shell_repo_base) + $stdout.puts 'gitlab-shell self-check successful'.color(:green) + else + $stdout.puts 'gitlab-shell self-check failed'.color(:red) + try_fixing_it( + 'Make sure GitLab is running;', + 'Check the gitlab-shell configuration file:', + sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}") + ) + fix_and_rerun + end + end + + # Helper methods + ######################## + + def gitlab_shell_path + Gitlab.config.gitlab_shell.path + end + + def gitlab_shell_version + Gitlab::Shell.new.version + end + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index 6227e461d24..07d479848fe 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module Helpers include ::Gitlab::TaskHelpers diff --git a/lib/system_check/incoming_email/foreman_configured_check.rb b/lib/system_check/incoming_email/foreman_configured_check.rb index 1db7bf2b782..944913087da 100644 --- a/lib/system_check/incoming_email/foreman_configured_check.rb +++ b/lib/system_check/incoming_email/foreman_configured_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module IncomingEmail class ForemanConfiguredCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index e55bea86d3f..613c2296375 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module IncomingEmail class ImapAuthenticationCheck < SystemCheck::BaseCheck @@ -7,7 +9,7 @@ module SystemCheck if config try_connect_imap else - @error = "#{mail_room_config_path} does not have mailboxes setup" + @error = "#{mail_room_config_path} does not have mailboxes set up" false end end diff --git a/lib/system_check/incoming_email/initd_configured_check.rb b/lib/system_check/incoming_email/initd_configured_check.rb index ea23b8ef49c..acb4b5a9e74 100644 --- a/lib/system_check/incoming_email/initd_configured_check.rb +++ b/lib/system_check/incoming_email/initd_configured_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module IncomingEmail class InitdConfiguredCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/incoming_email/mail_room_running_check.rb b/lib/system_check/incoming_email/mail_room_running_check.rb index c1807501829..b7aead4624e 100644 --- a/lib/system_check/incoming_email/mail_room_running_check.rb +++ b/lib/system_check/incoming_email/mail_room_running_check.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck module IncomingEmail class MailRoomRunningCheck < SystemCheck::BaseCheck diff --git a/lib/system_check/incoming_email_check.rb b/lib/system_check/incoming_email_check.rb new file mode 100644 index 00000000000..155b6547595 --- /dev/null +++ b/lib/system_check/incoming_email_check.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:incoming_email:check rake task + class IncomingEmailCheck < BaseCheck + set_name 'Incoming Email:' + + def multi_check + if Gitlab.config.incoming_email.enabled + checks = [ + SystemCheck::IncomingEmail::ImapAuthenticationCheck + ] + + if Rails.env.production? + checks << SystemCheck::IncomingEmail::InitdConfiguredCheck + checks << SystemCheck::IncomingEmail::MailRoomRunningCheck + else + checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck + end + + SystemCheck.run('Reply by email', checks) + else + $stdout.puts 'Reply by email is disabled in config/gitlab.yml' + end + end + end +end diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb new file mode 100644 index 00000000000..619fb3cccb8 --- /dev/null +++ b/lib/system_check/ldap_check.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:ldap:check rake task + class LdapCheck < BaseCheck + set_name 'LDAP:' + + def multi_check + if Gitlab::Auth::LDAP::Config.enabled? + # Only show up to 100 results because LDAP directories can be very big. + # This setting only affects the `rake gitlab:check` script. + limit = ENV['LDAP_CHECK_LIMIT'] + limit = 100 if limit.blank? + + check_ldap(limit) + else + $stdout.puts 'LDAP is disabled in config/gitlab.yml' + end + end + + private + + def check_ldap(limit) + servers = Gitlab::Auth::LDAP::Config.providers + + servers.each do |server| + $stdout.puts "Server: #{server}" + + begin + Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| + check_ldap_auth(adapter) + + $stdout.puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + + users = adapter.users(adapter.config.uid, '*', limit) + users.each do |user| + $stdout.puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end + end + rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + $stdout.puts "Could not connect to the LDAP server: #{e.message}".color(:red) + end + end + end + + def check_ldap_auth(adapter) + auth = adapter.config.has_auth? + + message = if auth && adapter.ldap.bind + 'Success'.color(:green) + elsif auth + 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end + + $stdout.puts "LDAP authentication... #{message}" + end + end +end diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb index b5f443abe06..53b2d8fd5b3 100644 --- a/lib/system_check/orphans/namespace_check.rb +++ b/lib/system_check/orphans/namespace_check.rb @@ -1,16 +1,20 @@ +# frozen_string_literal: true + module SystemCheck module Orphans class NamespaceCheck < SystemCheck::BaseCheck set_name 'Orphaned namespaces:' def multi_check - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) - - orphans = (toplevel_namespace_dirs - existing_namespaces) - print_orphans(orphans, storage_name) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) + toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) + + orphans = (toplevel_namespace_dirs - existing_namespaces) + print_orphans(orphans, storage_name) + end end clear_namespaces! # releases memory when check finishes diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb index 5ef0b93ad08..33020417e95 100644 --- a/lib/system_check/orphans/repository_check.rb +++ b/lib/system_check/orphans/repository_check.rb @@ -1,20 +1,23 @@ +# frozen_string_literal: true + module SystemCheck module Orphans class RepositoryCheck < SystemCheck::BaseCheck set_name 'Orphaned repositories:' - attr_accessor :orphans def multi_check - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - storage_path = repository_storage.legacy_disk_path + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |storage_name, repository_storage| + storage_path = repository_storage.legacy_disk_path - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) + $stdout.puts + $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) - repositories = disk_repositories(storage_path) - orphans = (repositories - fetch_repositories(storage_name)) + repositories = disk_repositories(storage_path) + orphans = (repositories - fetch_repositories(storage_name)) - print_orphans(orphans, storage_name) + print_orphans(orphans, storage_name) + end end end diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb new file mode 100644 index 00000000000..cc32feb8604 --- /dev/null +++ b/lib/system_check/rake_task/app_task.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:app:check rake task + module AppTask + extend RakeTaskHelpers + + def self.name + 'GitLab App' + end + + def self.checks + [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck, + SystemCheck::App::GitUserDefaultSSHConfigCheck, + SystemCheck::App::ActiveUsersCheck + ] + end + end + end +end diff --git a/lib/system_check/rake_task/gitaly_task.rb b/lib/system_check/rake_task/gitaly_task.rb new file mode 100644 index 00000000000..0c3f694f98a --- /dev/null +++ b/lib/system_check/rake_task/gitaly_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:gitaly:check rake task + class GitalyTask + extend RakeTaskHelpers + + def self.name + 'Gitaly' + end + + def self.checks + [SystemCheck::GitalyCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/gitlab_shell_task.rb b/lib/system_check/rake_task/gitlab_shell_task.rb new file mode 100644 index 00000000000..120e984c68b --- /dev/null +++ b/lib/system_check/rake_task/gitlab_shell_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:gitlab_shell:check rake task + class GitlabShellTask + extend RakeTaskHelpers + + def self.name + 'GitLab Shell' + end + + def self.checks + [SystemCheck::GitlabShellCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/gitlab_task.rb b/lib/system_check/rake_task/gitlab_task.rb new file mode 100644 index 00000000000..7ff36fd6eb5 --- /dev/null +++ b/lib/system_check/rake_task/gitlab_task.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:check rake task + class GitlabTask + extend RakeTaskHelpers + + def self.name + 'GitLab' + end + + def self.manual_run_checks! + start_checking("#{name} subtasks") + + subtasks.each(&:run_checks!) + + finished_checking("#{name} subtasks") + end + + def self.subtasks + [ + SystemCheck::RakeTask::GitlabShellTask, + SystemCheck::RakeTask::GitalyTask, + SystemCheck::RakeTask::SidekiqTask, + SystemCheck::RakeTask::IncomingEmailTask, + SystemCheck::RakeTask::LdapTask, + SystemCheck::RakeTask::AppTask + ] + end + end + end +end diff --git a/lib/system_check/rake_task/incoming_email_task.rb b/lib/system_check/rake_task/incoming_email_task.rb new file mode 100644 index 00000000000..c296c46feab --- /dev/null +++ b/lib/system_check/rake_task/incoming_email_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:incoming_email:check rake task + class IncomingEmailTask + extend RakeTaskHelpers + + def self.name + 'Incoming Email' + end + + def self.checks + [SystemCheck::IncomingEmailCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/ldap_task.rb b/lib/system_check/rake_task/ldap_task.rb new file mode 100644 index 00000000000..03a180b9dfb --- /dev/null +++ b/lib/system_check/rake_task/ldap_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:ldap:check rake task + class LdapTask + extend RakeTaskHelpers + + def self.name + 'LDAP' + end + + def self.checks + [SystemCheck::LdapCheck] + end + end + end +end diff --git a/lib/system_check/rake_task/orphans/namespace_task.rb b/lib/system_check/rake_task/orphans/namespace_task.rb new file mode 100644 index 00000000000..2822da45bc1 --- /dev/null +++ b/lib/system_check/rake_task/orphans/namespace_task.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + module Orphans + # Used by gitlab:orphans:check_namespaces rake task + class NamespaceTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [SystemCheck::Orphans::NamespaceCheck] + end + end + end + end +end diff --git a/lib/system_check/rake_task/orphans/repository_task.rb b/lib/system_check/rake_task/orphans/repository_task.rb new file mode 100644 index 00000000000..f14b3af22e8 --- /dev/null +++ b/lib/system_check/rake_task/orphans/repository_task.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + module Orphans + # Used by gitlab:orphans:check_repositories rake task + class RepositoryTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [SystemCheck::Orphans::RepositoryCheck] + end + end + end + end +end diff --git a/lib/system_check/rake_task/orphans_task.rb b/lib/system_check/rake_task/orphans_task.rb new file mode 100644 index 00000000000..31f8ede25e0 --- /dev/null +++ b/lib/system_check/rake_task/orphans_task.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:orphans:check rake task + class OrphansTask + extend RakeTaskHelpers + + def self.name + 'Orphans' + end + + def self.checks + [ + SystemCheck::Orphans::NamespaceCheck, + SystemCheck::Orphans::RepositoryCheck + ] + end + end + end +end diff --git a/lib/system_check/rake_task/rake_task_helpers.rb b/lib/system_check/rake_task/rake_task_helpers.rb new file mode 100644 index 00000000000..95f2a34a719 --- /dev/null +++ b/lib/system_check/rake_task/rake_task_helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Provides the run! method intended to be called from system check rake tasks + module RakeTaskHelpers + include ::SystemCheck::Helpers + + def run! + warn_user_is_not_gitlab + + if self.respond_to?(:manual_run_checks!) + manual_run_checks! + else + run_checks! + end + end + + def run_checks! + SystemCheck.run(name, checks) + end + + def name + raise NotImplementedError + end + + def checks + raise NotImplementedError + end + end + end +end diff --git a/lib/system_check/rake_task/sidekiq_task.rb b/lib/system_check/rake_task/sidekiq_task.rb new file mode 100644 index 00000000000..3ccf009d4b9 --- /dev/null +++ b/lib/system_check/rake_task/sidekiq_task.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module SystemCheck + module RakeTask + # Used by gitlab:sidekiq:check rake task + class SidekiqTask + extend RakeTaskHelpers + + def self.name + 'Sidekiq' + end + + def self.checks + [SystemCheck::SidekiqCheck] + end + end + end +end diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb new file mode 100644 index 00000000000..2f5fc2cea89 --- /dev/null +++ b/lib/system_check/sidekiq_check.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module SystemCheck + # Used by gitlab:sidekiq:check rake task + class SidekiqCheck < BaseCheck + set_name 'Sidekiq:' + + def multi_check + check_sidekiq_running + only_one_sidekiq_running + end + + private + + def check_sidekiq_running + $stdout.print "Running? ... " + + if sidekiq_process_count > 0 + $stdout.puts "yes".color(:green) + else + $stdout.puts "no".color(:red) + try_fixing_it( + sudo_gitlab("RAILS_ENV=production bin/background_jobs start") + ) + for_more_information( + see_installation_guide_section("Install Init Script"), + "see log/sidekiq.log for possible errors" + ) + fix_and_rerun + end + end + + def only_one_sidekiq_running + process_count = sidekiq_process_count + return if process_count.zero? + + $stdout.print 'Number of Sidekiq processes ... ' + + if process_count == 1 + $stdout.puts '1'.color(:green) + else + $stdout.puts "#{process_count}".color(:red) + try_fixing_it( + 'sudo service gitlab stop', + "sudo pkill -u #{gitlab_user} -f sidekiq", + "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq", + 'sudo service gitlab start' + ) + fix_and_rerun + end + end + + def sidekiq_process_count + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index d268f501b4a..11818ae54f8 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module SystemCheck # Simple Executor is current default executor for GitLab # It is a simple port from display logic in the old check.rake @@ -43,7 +45,7 @@ module SystemCheck # # @param [SystemCheck::BaseCheck] check_klass def run_check(check_klass) - $stdout.print "#{check_klass.display_name} ... " + print_display_name(check_klass) check = check_klass.new @@ -60,18 +62,18 @@ module SystemCheck end if check.check? - $stdout.puts check_klass.check_pass.color(:green) + print_check_pass(check_klass) else - $stdout.puts check_klass.check_fail.color(:red) + print_check_failure(check_klass) if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' if check.repair! - $stdout.puts 'Success'.color(:green) + print_success return else - $stdout.puts 'Failed'.color(:red) + print_failure end end @@ -83,6 +85,26 @@ module SystemCheck private + def print_display_name(check_klass) + $stdout.print "#{check_klass.display_name} ... " + end + + def print_check_pass(check_klass) + $stdout.puts check_klass.check_pass.color(:green) + end + + def print_check_failure(check_klass) + $stdout.puts check_klass.check_fail.color(:red) + end + + def print_success + $stdout.puts 'Success'.color(:green) + end + + def print_failure + $stdout.puts 'Failed'.color(:red) + end + # Prints header content for the series of checks to be executed for this component # # @param [String] component name of the component relative to the checks being executed diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake deleted file mode 100644 index 4b4881cecb8..00000000000 --- a/lib/tasks/flay.rake +++ /dev/null @@ -1,9 +0,0 @@ -desc 'Code duplication analyze via flay' -task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` - - if output.include?("Similar code found") || output.include?("IDENTICAL code found") - puts output - exit 1 - end -end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c6942d22926..560a52053d8 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -86,7 +86,7 @@ namespace :gemojione do SPRITESHEET_WIDTH = 860 SPRITESHEET_HEIGHT = 840 - # Setup a map to rename image files + # Set up a map to rename image files emoji_unicode_string_to_name_map = {} Gitlab::Emoji.emojis.each do |name, emoji_hash| # Ignore aliases diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 247d7be7d78..2235a6ba194 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -4,7 +4,7 @@ namespace :gettext do # Customize list of translatable files # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate - folders = %W(app lib config #{locale_path}).join(',') + folders = %W(ee app lib config #{locale_path}).join(',') exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( @@ -16,10 +16,32 @@ namespace :gettext do # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) - Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end + task :regenerate do + pot_file = 'locale/gitlab.pot' + # Remove all translated files, this speeds up finding + FileUtils.rm Dir['locale/**/gitlab.*'] + # remove the `pot` file to ensure it's completely regenerated + FileUtils.rm_f pot_file + + Rake::Task['gettext:find'].invoke + + # leave only the required changes. + `git checkout -- locale/*/gitlab.po` + + # Remove timestamps from the pot file + pot_content = File.read pot_file + pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '') + File.write pot_file, pot_content + + puts <<~MSG + All done. Please commit the changes to `locale/gitlab.pot`. + + MSG + end + desc 'Lint all po files in `locale/' task lint: :environment do require 'simple_po_parser' @@ -50,6 +72,40 @@ namespace :gettext do end end + task :updated_check do + pot_file = 'locale/gitlab.pot' + # Removing all pre-translated files speeds up `gettext:find` as the + # files don't need to be merged. + # Having `LC_MESSAGES/gitlab.mo files present also confuses the output. + FileUtils.rm Dir['locale/**/gitlab.*'] + FileUtils.rm_f pot_file + + # `gettext:find` writes touches to temp files to `stderr` which would cause + # `static-analysis` to report failures. We can ignore these. + silence_stderr do + Rake::Task['gettext:find'].invoke + end + + pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip + + # reset the locale folder for potential next tasks + `git checkout -- locale` + + if pot_diff.present? + raise <<~MSG + Newly translated strings found, please add them to `#{pot_file}` by running: + + bin/rake gettext:regenerate + + Then commit and push the resulting changes to `#{pot_file}`. + + The diff was: + + #{pot_diff} + MSG + end + end + def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" @@ -62,4 +118,15 @@ namespace :gettext do end end end + + def silence_stderr(&block) + old_stderr = $stderr.dup + $stderr.reopen(File::NULL) + $stderr.sync = true + + yield + ensure + $stderr.reopen(old_stderr) + old_stderr.close + end end diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index bfca4bfb3f7..e7634d2ed4f 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) - logger.info("Transferred artifacts of #{build.id} of #{build.artifacts_size} to object storage") + logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") rescue => e logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") end diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 83dd870fa31..c0d6cc8ca8e 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -1,6 +1,6 @@ namespace :gitlab do namespace :import do - desc "GitLab | Add all users to all projects (admin users are added as masters)" + desc "GitLab | Add all users to all projects (admin users are added as maintainers)" task all_users_to_all_projects: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) @@ -10,11 +10,11 @@ namespace :gitlab do ProjectMember.add_users_to_projects(project_ids, user_ids, ProjectMember::DEVELOPER) puts "Importing #{admin_ids.size} admins into #{project_ids.size} projects" - ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MASTER) + ProjectMember.add_users_to_projects(project_ids, admin_ids, ProjectMember::MAINTAINER) end desc "GitLab | Add a specific user to all projects (as a developer)" - task :user_to_projects, [:email] => :environment do |t, args| + task :user_to_projects, [:email] => :environment do |t, args| user = User.find_by(email: args.email) project_ids = Project.pluck(:id) puts "Importing #{user.email} users into #{project_ids.size} projects" @@ -22,7 +22,7 @@ namespace :gitlab do end desc "GitLab | Add all users to all groups (admin users are added as owners)" - task all_users_to_all_groups: :environment do |t, args| + task all_users_to_all_groups: :environment do |t, args| user_ids = User.where(admin: false).pluck(:id) admin_ids = User.where(admin: true).pluck(:id) groups = Group.all @@ -36,7 +36,7 @@ namespace :gitlab do end desc "GitLab | Add a specific user to all groups (as a developer)" - task :user_to_groups, [:email] => :environment do |t, args| + task :user_to_groups, [:email] => :environment do |t, args| user = User.find_by_email args.email groups = Group.all puts "Importing #{user.email} users into #{groups.size} groups" diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index c04dae7446f..b594f150c3b 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,454 +1,66 @@ namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' - task check: %w{gitlab:gitlab_shell:check - gitlab:sidekiq:check - gitlab:incoming_email:check - gitlab:ldap:check - gitlab:app:check} + task check: :gitlab_environment do + SystemCheck::RakeTask::GitlabTask.run! + end namespace :app do desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :gitlab_environment do - warn_user_is_not_gitlab - - checks = [ - SystemCheck::App::GitConfigCheck, - SystemCheck::App::DatabaseConfigExistsCheck, - SystemCheck::App::MigrationsAreUpCheck, - SystemCheck::App::OrphanedGroupMembersCheck, - SystemCheck::App::GitlabConfigExistsCheck, - SystemCheck::App::GitlabConfigUpToDateCheck, - SystemCheck::App::LogWritableCheck, - SystemCheck::App::TmpWritableCheck, - SystemCheck::App::UploadsDirectoryExistsCheck, - SystemCheck::App::UploadsPathPermissionCheck, - SystemCheck::App::UploadsPathTmpPermissionCheck, - SystemCheck::App::InitScriptExistsCheck, - SystemCheck::App::InitScriptUpToDateCheck, - SystemCheck::App::ProjectsHaveNamespaceCheck, - SystemCheck::App::RedisVersionCheck, - SystemCheck::App::RubyVersionCheck, - SystemCheck::App::GitVersionCheck, - SystemCheck::App::GitUserDefaultSSHConfigCheck, - SystemCheck::App::ActiveUsersCheck - ] - - SystemCheck.run('GitLab', checks) + SystemCheck::RakeTask::AppTask.run! end end namespace :gitlab_shell do desc "GitLab | Check the configuration of GitLab Shell" task check: :gitlab_environment do - warn_user_is_not_gitlab - start_checking "GitLab Shell" - - check_gitlab_shell - check_repo_base_exists - check_repo_base_is_not_symlink - check_repo_base_user_and_group - check_repo_base_permissions - check_repos_hooks_directory_is_link - check_gitlab_shell_self_test - - finished_checking "GitLab Shell" - end - - # Checks - ######################## - - def check_repo_base_exists - puts "Repo base directory exists?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - if File.exist?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_is_not_symlink - puts "Repo storage directories are symlinks?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun - end - end - end - - def check_repo_base_permissions - puts "Repo paths access is drwxrws---?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_user_and_group - gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - user_id = uid_for(gitlab_shell_ssh_user) - root_group_id = gid_for('root') - group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] - if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repos_hooks_directory_is_link - print "hooks directories in repos are links: ... " - - gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - project_hook_directory = File.join(project.repository.path_to_repo, "hooks") - - if project.empty_repo? - puts "repository is empty".color(:magenta) - elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) && - (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path)) - puts 'ok'.color(:green) - else - puts "wrong or missing hooks".color(:red) - try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), - 'Check the hooks_path in config/gitlab.yml', - 'Check your gitlab-shell installation' - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_gitlab_shell_self_test - gitlab_shell_repo_base = gitlab_shell_path - check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) - puts "Running #{check_cmd}" - - if system(check_cmd, chdir: gitlab_shell_repo_base) - puts 'gitlab-shell self-check successful'.color(:green) - else - puts 'gitlab-shell self-check failed'.color(:red) - try_fixing_it( - 'Make sure GitLab is running;', - 'Check the gitlab-shell configuration file:', - sudo_gitlab("editor #{File.expand_path('config.yml', gitlab_shell_repo_base)}") - ) - fix_and_rerun - end - end - - # Helper methods - ######################## - - def gitlab_shell_path - Gitlab.config.gitlab_shell.path - end - - def gitlab_shell_version - Gitlab::Shell.new.version - end - - def gitlab_shell_major_version - Gitlab::Shell.version_required.split('.')[0].to_i - end - - def gitlab_shell_minor_version - Gitlab::Shell.version_required.split('.')[1].to_i + SystemCheck::RakeTask::GitlabShellTask.run! end + end - def gitlab_shell_patch_version - Gitlab::Shell.version_required.split('.')[2].to_i + namespace :gitaly do + desc 'GitLab | Check the health of Gitaly' + task check: :gitlab_environment do + SystemCheck::RakeTask::GitalyTask.run! end end namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :gitlab_environment do - warn_user_is_not_gitlab - start_checking "Sidekiq" - - check_sidekiq_running - only_one_sidekiq_running - - finished_checking "Sidekiq" - end - - # Checks - ######################## - - def check_sidekiq_running - print "Running? ... " - - if sidekiq_process_count > 0 - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("RAILS_ENV=production bin/background_jobs start") - ) - for_more_information( - see_installation_guide_section("Install Init Script"), - "see log/sidekiq.log for possible errors" - ) - fix_and_rerun - end - end - - def only_one_sidekiq_running - process_count = sidekiq_process_count - return if process_count.zero? - - print 'Number of Sidekiq processes ... ' - - if process_count == 1 - puts '1'.color(:green) - else - puts "#{process_count}".color(:red) - try_fixing_it( - 'sudo service gitlab stop', - "sudo pkill -u #{gitlab_user} -f sidekiq", - "sleep 10 && sudo pkill -9 -u #{gitlab_user} -f sidekiq", - 'sudo service gitlab start' - ) - fix_and_rerun - end - end - - def sidekiq_process_count - ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count + SystemCheck::RakeTask::SidekiqTask.run! end end namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" task check: :gitlab_environment do - warn_user_is_not_gitlab - - if Gitlab.config.incoming_email.enabled - checks = [ - SystemCheck::IncomingEmail::ImapAuthenticationCheck - ] - - if Rails.env.production? - checks << SystemCheck::IncomingEmail::InitdConfiguredCheck - checks << SystemCheck::IncomingEmail::MailRoomRunningCheck - else - checks << SystemCheck::IncomingEmail::ForemanConfiguredCheck - end - - SystemCheck.run('Reply by email', checks) - else - puts 'Reply by email is disabled in config/gitlab.yml' - end + SystemCheck::RakeTask::IncomingEmailTask.run! end end namespace :ldap do task :check, [:limit] => :gitlab_environment do |_, args| - # Only show up to 100 results because LDAP directories can be very big. - # This setting only affects the `rake gitlab:check` script. - args.with_defaults(limit: 100) - warn_user_is_not_gitlab - start_checking "LDAP" - - if Gitlab::Auth::LDAP::Config.enabled? - check_ldap(args.limit) - else - puts 'LDAP is disabled in config/gitlab.yml' - end - - finished_checking "LDAP" - end - - def check_ldap(limit) - servers = Gitlab::Auth::LDAP::Config.providers - - servers.each do |server| - puts "Server: #{server}" + ENV['LDAP_CHECK_LIMIT'] = args.limit if args.limit.present? - begin - Gitlab::Auth::LDAP::Adapter.open(server) do |adapter| - check_ldap_auth(adapter) - - puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" - - users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" - end - end - rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e - puts "Could not connect to the LDAP server: #{e.message}".color(:red) - end - end - end - - def check_ldap_auth(adapter) - auth = adapter.config.has_auth? - - message = if auth && adapter.ldap.bind - 'Success'.color(:green) - elsif auth - 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) - else - 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) - end - - puts "LDAP authentication... #{message}" - end - end - - namespace :repo do - desc "GitLab | Check the integrity of the repositories managed by GitLab" - task check: :gitlab_environment do - puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) - Rake::Task["gitlab:git:fsck"].execute + SystemCheck::RakeTask::LdapTask.run! end end namespace :orphans do desc 'Gitlab | Check for orphaned namespaces and repositories' task check: :gitlab_environment do - warn_user_is_not_gitlab - checks = [ - SystemCheck::Orphans::NamespaceCheck, - SystemCheck::Orphans::RepositoryCheck - ] - - SystemCheck.run('Orphans', checks) + SystemCheck::RakeTask::OrphansTask.run! end desc 'GitLab | Check for orphaned namespaces in the repositories path' task check_namespaces: :gitlab_environment do - warn_user_is_not_gitlab - checks = [SystemCheck::Orphans::NamespaceCheck] - - SystemCheck.run('Orphans', checks) + SystemCheck::RakeTask::Orphans::NamespaceTask.run! end desc 'GitLab | Check for orphaned repositories in the repositories path' task check_repositories: :gitlab_environment do - warn_user_is_not_gitlab - checks = [SystemCheck::Orphans::RepositoryCheck] - - SystemCheck.run('Orphans', checks) - end - end - - namespace :user do - desc "GitLab | Check the integrity of a specific user's repositories" - task :check_repos, [:username] => :gitlab_environment do |t, args| - username = args[:username] || prompt("Check repository integrity for username? ".color(:blue)) - user = User.find_by(username: username) - if user - repo_dirs = user.authorized_projects.map do |p| - p.repository.path_to_repo - end - - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - else - puts "\nUser '#{username}' not found".color(:red) - end - end - end - - # Helper methods - ########################## - - def check_gitlab_shell - required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) - current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) - - print "GitLab Shell version >= #{required_version} ? ... " - if current_version.valid? && required_version <= current_version - puts "OK (#{current_version})".color(:green) - else - puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) + SystemCheck::RakeTask::Orphans::RepositoryTask.run! end end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index d6d15285489..451ba651674 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,41 +1,29 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954 -# +# frozen_string_literal: true +require 'set' + namespace :gitlab do namespace :cleanup do - HASHED_REPOSITORY_NAME = '@hashed'.freeze - desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do - warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] + namespaces = Set.new(Namespace.pluck(:path)) + namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX - namespaces = Namespace.pluck(:path) - namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored - Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = repository_storage.legacy_disk_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitaly::Server.all.each do |server| + all_dirs = Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories(depth: 0) + .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) } - puts git_base_path.color(:yellow) puts "Looking for directories to remove... " - - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end - - all_dirs.reject! do |dir| - dir_name = File.basename dir - - # skip if namespace present - namespaces.include?(dir_name) - end - all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) - else - puts "Cannot remove #{dir_path}".color(:red) + if remove? + begin + Gitlab::GitalyClient::NamespaceService.new(server.storage) + .remove(dir_path) + + puts "Removed...#{dir_path}" + rescue StandardError => e + puts "Cannot remove #{dir_path}: #{e.message}".color(:red) end else puts "Can be removed: #{dir_path}".color(:red) @@ -43,35 +31,36 @@ namespace :gitlab do end end - unless remove_flag + unless remove? puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) end end desc "GitLab | Cleanup | Clean repositories" task repos: :gitlab_environment do - warn_user_is_not_gitlab - move_suffix = "+orphaned+#{Time.now.to_i}" - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = repository_storage.legacy_disk_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path - .sub(repo_root, '') - .sub(%r{^/*}, '') - .chomp('.git') - .chomp('.wiki') - - # TODO ignoring hashed repositories for now. But revisit to fully support - # possible orphaned hashed repos - next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) - - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + + Gitaly::Server.all.each do |server| + Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories + .each do |path| + repo_with_namespace = path.chomp('.git').chomp('.wiki') + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX) + next if Project.find_by_full_path(repo_with_namespace) + + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + + begin + Gitlab::GitalyClient::NamespaceService + .new(server.storage) + .rename(path, new_path) + rescue StandardError => e + puts "Error occured while moving the repository: #{e.message}".color(:red) end end end @@ -104,27 +93,46 @@ namespace :gitlab do end end - # This is a rake task which removes faulty refs. These refs where only - # created in the 8.13.RC cycle, and fixed in the stable builds which were - # released. So likely this should only be run once on gitlab.com - # Faulty refs are moved so they are kept around, else some features break. - desc 'GitLab | Cleanup | Remove faulty deployment refs' - task move_faulty_deployment_refs: :gitlab_environment do - projects = Project.where(id: Deployment.select(:project_id).distinct) + desc "GitLab | Cleanup | Clean orphaned project uploads" + task project_uploads: :gitlab_environment do + warn_user_is_not_gitlab - projects.find_each do |project| - rugged = project.repository.rugged + cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger) + cleaner.run!(dry_run: dry_run?) - max_iid = project.deployments.maximum(:iid) + if dry_run? + logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow) + end + end - rugged.references.each('refs/environments/**/*') do |ref| - id = ref.name.split('/').last.to_i - next unless id > max_iid + desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db' + task remote_upload_files: :environment do + cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger) + cleaner.run!(dry_run: dry_run?) - project.deployments.find(id).create_ref - project.repository.delete_refs(ref) - end + if dry_run? + logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow) end end + + def remove? + ENV['REMOVE'] == 'true' + end + + def dry_run? + ENV['DRY_RUN'] != 'false' + end + + def logger + return @logger if defined?(@logger) + + @logger = if Rails.env.development? || Rails.env.production? + Logger.new(STDOUT).tap do |stdout_logger| + stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger)) + end + else + Rails.logger + end + end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 139ab70e125..74cd70c6e9f 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -46,9 +46,13 @@ namespace :gitlab do desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do - if ActiveRecord::Base.connection.tables.any? + # Check if we have existing db tables + # The schema_migrations table will still exist if drop_tables was called + if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else + # Add post-migrate paths to ensure we mark all migrations as up + Gitlab::Database.add_post_migrate_path_to_rails(force: true) Rake::Task['db:schema:load'].invoke Rake::Task['db:seed_fu'].invoke end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cb4f7e5c8a8..8a53b51d4fe 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,87 +1,24 @@ namespace :gitlab do namespace :git do - desc "GitLab | Git | Repack" - task repack: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - desc "GitLab | Git | Run garbage collection on all repos" - task gc: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - desc "GitLab | Git | Prune all repos" - task prune: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - desc 'GitLab | Git | Check all repos integrity' task fsck: :gitlab_environment do - failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| - check_config_lock(repo) - check_ref_locks(repo) - end - - if failures.empty? - puts "Done".color(:green) - else - output_failures(failures) - end - end - - def perform_git_cmd(cmd, message) - puts "Starting #{message} on all repositories" - failures = [] - all_repos do |repo| - if system(*cmd, chdir: repo) - puts "Performed #{message} at #{repo}" - else - failures << repo + Project.find_each(batch_size: 100) do |project| + begin + project.repository.fsck + + rescue => e + failures << "#{project.full_path} on #{project.repository_storage}: #{e}" end - yield(repo) if block_given? + puts "Performed integrity check for #{project.repository.full_path}" end - failures - end - - def output_failures(failures) - puts "The following repositories reported errors:".color(:red) - failures.each { |f| puts "- #{f}" } - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - - if lock_files.present? - puts "Ref lock files exist:".color(:red) - - lock_files.each { |lock_file| puts " #{lock_file}" } + if failures.empty? + puts "Done".color(:green) else - puts "No ref lock files exist".color(:green) + puts "The following repositories reported errors:".color(:red) + failures.each { |f| puts "- #{f}" } end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index e9ca6404fe8..80de3d2ef51 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,13 +1,12 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir, :repo] => :gitlab_environment do |t, args| - require 'toml-rb' - + task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab - unless args.dir.present? - abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") + unless args.dir.present? && args.storage_path.present? + abort %(Please specify the directory where you want to install gitaly and the path for the default storage +Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") end args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') @@ -27,7 +26,8 @@ namespace :gitlab do "BUNDLE_PATH=#{Bundler.bundle_path}") end - Gitlab::SetupHelper.create_gitaly_configuration(args.dir) + storage_paths = { 'default' => args.storage_path } + Gitlab::SetupHelper.create_gitaly_configuration(args.dir, storage_paths) Dir.chdir(args.dir) do # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? @@ -35,17 +35,5 @@ namespace :gitlab do end end end - - desc "GitLab | Print storage configuration in TOML format" - task storage_config: :environment do - require 'toml-rb' - - puts "# Gitaly storage configuration generated from #{Gitlab.config.source} on #{Time.current.to_s(:long)}" - puts "# This is in TOML format suitable for use in Gitaly's config.toml file." - - # Exclude gitaly-ruby configuration because that depends on the gitaly - # installation directory. - puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) - end end end diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 289aa5d9060..e97d77d20e0 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -54,8 +54,8 @@ namespace :gitlab do puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}" - puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}" - puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled + puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}" + puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled? # check Gitolite version gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION" @@ -67,8 +67,10 @@ namespace :gitlab do puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" - Gitlab.config.repositories.storages.each do |name, repository_storage| - puts "- #{name}: \t#{repository_storage.legacy_disk_path}" + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage.legacy_disk_path}" + end end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake index c66a2a263dc..0459de27c96 100644 --- a/lib/tasks/gitlab/ldap.rake +++ b/lib/tasks/gitlab/ldap.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :ldap do desc 'GitLab | LDAP | Rename provider' - task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + task :rename_provider, [:old_provider, :new_provider] => :gitlab_environment do |_, args| old_provider = args[:old_provider] || prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) new_provider = args[:new_provider] || diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 4fcbbbf8c9d..0ebc6f00793 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -92,9 +92,11 @@ namespace :gitlab do def setup warn_user_is_not_gitlab + ensure_write_to_authorized_keys_is_enabled + unless ENV['force'] == 'yes' - puts "This will rebuild an authorized_keys file." - puts "You will lose any data stored in authorized_keys file." + puts "This task will now rebuild the authorized_keys file." + puts "You will lose any data stored in the authorized_keys file." ask_to_continue puts "" end @@ -118,4 +120,44 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end + + def ensure_write_to_authorized_keys_is_enabled + return if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + + puts authorized_keys_is_disabled_warning + + unless ENV['force'] == 'yes' + puts 'Do you want to permanently enable the "Write to authorized_keys file" setting now?' + ask_to_continue + end + + puts 'Enabling the "Write to authorized_keys file" setting...' + Gitlab::CurrentSettings.current_application_settings.update!(authorized_keys_enabled: true) + + puts 'Successfully enabled "Write to authorized_keys file"!' + puts '' + end + + def authorized_keys_is_disabled_warning + <<-MSG.strip_heredoc + WARNING + + The "Write to authorized_keys file" setting is disabled, which prevents + the file from being rebuilt! + + It should be enabled for most GitLab installations. Large installations + may wish to disable it as part of speeding up SSH operations. + + See https://docs.gitlab.com/ee/administration/operations/fast_ssh_key_lookup.html + + If you did not intentionally disable this option in Admin Area > Settings, + then you may have been affected by the 9.3.0 bug in which the new setting + was disabled by default. + + https://gitlab.com/gitlab-org/gitlab-ee/issues/2738 + + It was reverted in 9.3.1 and fixed in 9.3.3, however, if Settings were + saved while the setting was unchecked, then it is still disabled. + MSG + end end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 68d6f9d7cb1..f9ce3e1d338 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -2,11 +2,34 @@ namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do - legacy_projects_count = Project.with_unmigrated_storage.count + if Gitlab::Database.read_only? + warn 'This task requires database write access. Exiting.' + + next + end + + storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper + if helper.range_single_item? + project = Project.with_unmigrated_storage.find_by(id: helper.range_from) + + unless project + warn "There are no projects requiring storage migration with ID=#{helper.range_from}" + + next + end + + puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." + storage_migrator.migrate(project) + + next + end + + legacy_projects_count = Project.with_unmigrated_storage.count + if legacy_projects_count == 0 - puts 'There are no projects requiring storage migration. Nothing to do!' + warn 'There are no projects requiring storage migration. Nothing to do!' next end @@ -14,7 +37,7 @@ namespace :gitlab do print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" helper.project_id_batches do |start, finish| - StorageMigratorWorker.perform_async(start, finish) + storage_migrator.bulk_schedule(start: start, finish: finish) print '.' end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake index fd2a4f2d11a..5a232091a7e 100644 --- a/lib/tasks/gitlab/traces.rake +++ b/lib/tasks/gitlab/traces.rake @@ -8,9 +8,7 @@ namespace :gitlab do logger = Logger.new(STDOUT) logger.info('Archiving legacy traces') - Ci::Build.finished - .where('NOT EXISTS (?)', - Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id')) + Ci::Build.finished.without_archived_trace .order(id: :asc) .find_in_batches(batch_size: 1000) do |jobs| job_ids = jobs.map { |job| [job.id] } @@ -20,5 +18,22 @@ namespace :gitlab do logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") end end + + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of job traces') + + Ci::Build.joins(:project) + .with_archived_trace_stored_locally + .find_each(batch_size: 10) do |build| + begin + build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE) + + logger.info("Transferred job trace of #{build.id} to object storage") + rescue => e + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + end + end + end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..abe10f5580e 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end @@ -98,10 +100,6 @@ namespace :gitlab do /(\.{1,2}|LICENSE|Global|\.gitignore)\z/ ), Template.new( - "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ - ), - Template.new( "https://gitlab.com/gitlab-org/Dockerfile.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 78e18992a8e..1c93609a006 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -1,6 +1,30 @@ namespace :gitlab do namespace :uploads do - desc 'GitLab | Uploads | Migrate the uploaded files to object storage' + namespace :migrate do + desc "GitLab | Uploads | Migrate all uploaded files to object storage" + task all: :environment do + categories = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)] + + categories.each do |args| + Rake::Task["gitlab:uploads:migrate"].invoke(*args) + Rake::Task["gitlab:uploads:migrate"].reenable + end + end + end + + # The following is the actual rake task that migrates uploads of specified + # category to object storage + desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| batch_size = ENV.fetch('BATCH', 200).to_i @to_store = ObjectStorage::Store::REMOTE @@ -8,7 +32,7 @@ namespace :gitlab do @uploader_class = args.uploader_class.constantize @model_class = args.model_class.constantize - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) # rubocop: disable Cop/InBatches + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) end def enqueue_batch(batch, index) diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 5a1c8006052..15cec80b6a6 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -25,11 +25,22 @@ namespace :gitlab do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] - projects = find_projects(namespace_path) - project_ids = projects.pluck(:id) + web_hooks = find_web_hooks(namespace_path) puts "Removing webhooks with the url '#{web_hook_url}' ... " - count = WebHook.where(url: web_hook_url, project_id: project_ids, type: 'ProjectHook').delete_all + + # FIXME: Hook URLs are now encrypted, so there is no way to efficiently + # find them all in SQL. For now, check them in Ruby. If this is too slow, + # we could consider storing a hash of the URL alongside the encrypted + # value to speed up searches + count = 0 + web_hooks.find_each do |hook| + next unless hook.url == web_hook_url + + hook.destroy! + count += 1 + end + puts "#{count} webhooks were removed." end @@ -37,29 +48,37 @@ namespace :gitlab do task list: :environment do namespace_path = ENV['NAMESPACE'] - projects = find_projects(namespace_path) - web_hooks = projects.all.map(&:hooks).flatten - web_hooks.each do |hook| + web_hooks = find_web_hooks(namespace_path) + web_hooks.find_each do |hook| puts "#{hook.project.name.truncate(20).ljust(20)} -> #{hook.url}" end - puts "\n#{web_hooks.size} webhooks found." + puts "\n#{web_hooks.count} webhooks found." end end def find_projects(namespace_path) if namespace_path.blank? Project - elsif namespace_path == '/' - Project.in_namespace(nil) else - namespace = Namespace.where(path: namespace_path).first - if namespace - Project.in_namespace(namespace.id) - else + namespace = Namespace.find_by_full_path(namespace_path) + + unless namespace puts "Namespace not found: #{namespace_path}".color(:red) exit 2 end + + Project.in_namespace(namespace.id) + end + end + + def find_web_hooks(namespace_path) + if namespace_path.blank? + ProjectHook + else + project_ids = find_projects(namespace_path).select(:id) + + ProjectHook.where(project_id: project_ids) end end end diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index ad2d034b0b4..786efd14b1a 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -2,5 +2,16 @@ unless Rails.env.production? require 'haml_lint/rake_task' require 'haml_lint/inline_javascript' + # Workaround for warnings from parser/current + # Keep it even if it no longer emits any warnings, + # because we'll still see warnings in console/server anyway, + # and we don't need to break static-analysis for this. + task :haml_lint do + require 'parser' + def Parser.warn(*args) + puts(*args) # static-analysis ignores stdout if status is 0 + end + end + HamlLint::RakeTask.new end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index aafbe52e5f8..f912f521dfb 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -9,7 +9,10 @@ class GithubImport def initialize(token, gitlab_username, project_path, extras) @options = { token: token } @project_path = project_path - @current_user = User.find_by_username(gitlab_username) + @current_user = UserFinder.new(gitlab_username).find_by_username + + raise "GitLab user #{gitlab_username} not found. Please specify a valid username." unless @current_user + @github_repo = extras.empty? ? nil : extras.first end @@ -39,7 +42,7 @@ class GithubImport end def import! - @project.force_import_start + @project.import_state.force_start import_success = false @@ -50,11 +53,11 @@ class GithubImport end if import_success - @project.import_finish + @project.after_import puts "Import finished. Timings: #{timings}".color(:green) else puts "Import was not successful. Errors were as follows:" - puts @project.import_error + puts @project.import_state.last_error end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 8b86a5c72a5..5d673a1a285 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -17,16 +17,25 @@ unless Rails.env.production? Rake::Task['eslint'].invoke end + desc "GitLab | lint | Lint HAML files" + task :haml do + begin + Rake::Task['haml_lint'].invoke + rescue RuntimeError # The haml_lint tasks raise a RuntimeError + exit(1) + end + end + desc "GitLab | lint | Run several lint checks" task :all do status = 0 %w[ config_lint - haml_lint + lint:haml scss_lint - flay gettext:lint + gettext:updated_check lint:static_verification ].each do |task| pid = Process.fork do @@ -38,13 +47,12 @@ unless Rails.env.production? $stderr.reopen(wr_err) begin - begin - Rake::Task[task].invoke - rescue RuntimeError # The haml_lint tasks raise a RuntimeError - exit(1) - end + Rake::Task[task].invoke rescue SystemExit => ex - msg = "*** Rake task #{task} failed with the following error(s):" + msg = "*** Rake task #{task} exited:" + raise ex + rescue => ex + msg = "*** Rake task #{task} raised #{ex.class}:" raise ex ensure $stdout.reopen(stdout) diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 9b05876034c..c77fa49d586 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -3,6 +3,7 @@ require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') +require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do @@ -12,4 +13,5 @@ task add_limits_mysql: :environment do MergeRequestDiffFileLimitsToMysql.new.up LimitsCiBuildTraceChunksRawDataForMysql.new.up IncreaseMysqlTextLimitForGpgKeys.new.up + PrometheusMetricsLimitsToMysql.new.up end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index e7aab50e42a..f69d204c579 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -22,3 +22,18 @@ task setup_postgresql: :environment do ProjectNameLowerIndex.new.up AddPathIndexToRedirectRoutes.new.up end + +desc 'GitLab | Generate PostgreSQL Password Hash' +task :postgresql_md5_hash do + require 'digest' + username = ENV.fetch('USERNAME') do |missing| + puts "You must provide an username with '#{missing}' ENV variable" + exit(1) + end + password = ENV.fetch('PASSWORD') do |missing| + puts "You must provide a password with '#{missing}' ENV variable" + exit(1) + end + hash = Digest::MD5.hexdigest("#{password}#{username}") + puts "The MD5 hash of your database password for user: #{username} -> #{hash}" +end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 81829668de8..eec024f9bbb 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,4 +1,7 @@ require_relative '../../app/models/concerns/token_authenticatable.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb' namespace :tokens do desc "Reset all GitLab incoming email tokens" @@ -26,13 +29,6 @@ class TmpUser < ActiveRecord::Base self.table_name = 'users' - def reset_incoming_email_token! - write_new_token(:incoming_email_token) - save!(validate: false) - end - - def reset_feed_token! - write_new_token(:feed_token) - save!(validate: false) - end + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } + add_authentication_token_field :feed_token end diff --git a/lib/unfold_form.rb b/lib/unfold_form.rb index fcd01503d1b..05bb3ed7f1c 100644 --- a/lib/unfold_form.rb +++ b/lib/unfold_form.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative 'gt_one_coercion' class UnfoldForm diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 5dc85b2baea..aae542f02ac 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "tempfile" require "tmpdir" require "fileutils" @@ -21,14 +23,14 @@ class UploadedFile raise InvalidPathError, "#{path} file does not exist" unless ::File.exist?(path) @content_type = content_type - @original_filename = filename || ::File.basename(path) + @original_filename = sanitize_filename(filename || path) @content_type = content_type @sha256 = sha256 @remote_id = remote_id @tempfile = File.new(path, 'rb') end - def self.from_params(params, field, upload_path) + def self.from_params(params, field, upload_paths) unless params["#{field}.path"] raise InvalidPathError, "file is invalid" if params["#{field}.remote_id"] @@ -37,7 +39,8 @@ class UploadedFile file_path = File.realpath(params["#{field}.path"]) - unless self.allowed_path?(file_path, [upload_path, Dir.tmpdir].compact) + paths = Array(upload_paths) << Dir.tmpdir + unless self.allowed_path?(file_path, paths.compact) raise InvalidPathError, "insecure path used '#{file_path}'" end @@ -54,6 +57,16 @@ class UploadedFile end end + # copy-pasted from CarrierWave::SanitizedFile + def sanitize_filename(name) + name = name.tr("\\", "/") # work-around for IE + name = ::File.basename(name) + name = name.gsub(CarrierWave::SanitizedFile.sanitize_regexp, "_") + name = "_#{name}" if name =~ /\A\.+\z/ + name = "unnamed" if name.empty? + name.mb_chars.to_s + end + def path @tempfile.path end diff --git a/lib/version_check.rb b/lib/version_check.rb index 91ad07feee5..c9f102f6b19 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -1,18 +1,21 @@ +# frozen_string_literal: true + require "base64" # This class is used to build image URL to # check if it is a new version for update class VersionCheck - def data + def self.data { version: Gitlab::VERSION } end - def url + def self.url encoded_data = Base64.urlsafe_encode64(data.to_json) + "#{host}?gitlab_info=#{encoded_data}" end - def host + def self.host 'https://version.gitlab.com/check.svg' end end |