diff options
Diffstat (limited to 'lib')
181 files changed, 2997 insertions, 925 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 1bf20f76ad6..52cd7cbe3db 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,6 +6,7 @@ module API version 'v3', using: :path do helpers ::API::V3::Helpers + helpers ::API::Helpers::CommonHelpers mount ::API::V3::AwardEmoji mount ::API::V3::Boards @@ -44,6 +45,9 @@ module API end before { allow_access_with_scope :api } + before { Gitlab::I18n.set_locale(current_user) } + + after { Gitlab::I18n.reset_locale } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -77,6 +81,7 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::SentryHelper helpers ::API::Helpers + helpers ::API::Helpers::CommonHelpers # Keep in alphabetical order mount ::API::AccessRequests diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 64ab6f01eb5..3fc2b453eb6 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -5,7 +5,10 @@ module API end class UserBasic < UserSafe - expose :id, :state, :avatar_url + expose :id, :state + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -14,7 +17,6 @@ module API class User < UserBasic expose :created_at - expose :admin?, as: :is_admin expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end @@ -41,8 +43,9 @@ module API expose :external end - class UserWithPrivateToken < UserPublic + class UserWithPrivateDetails < UserPublic expose :private_token + expose :admin?, as: :is_admin end class Email < Grape::Entity @@ -50,7 +53,7 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events expose :enable_ssl_verification end @@ -97,7 +100,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -141,7 +146,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -256,7 +263,11 @@ module API class IssueBasic < ProjectEntity expose :label_names, as: :labels expose :milestone, using: Entities::Milestone - expose :assignee, :author, using: Entities::UserBasic + expose :assignees, :author, using: Entities::UserBasic + + expose :assignee, using: ::API::Entities::UserBasic do |issue, options| + issue.assignees.first + end expose :user_notes_count expose :upvotes, :downvotes diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 09d105f6b4c..3da7d735da8 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -24,7 +24,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -52,7 +52,7 @@ module API elsif current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ddff3c8c1e8..226a7ddd50e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -91,8 +91,8 @@ module API end def find_project_snippet(id) - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params).find(id) + finder_params = { project: user_project } + SnippetsFinder.new(current_user, finder_params).execute.find(id) end def find_merge_request_with_access(iid, access_level = :read_merge_request) @@ -102,7 +102,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(current_user, :access_api) + unauthorized! unless current_user && can?(initial_current_user, :access_api) end def authenticate_non_get! @@ -301,7 +301,7 @@ module API UploadedFile.new( file_path, params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', + params["#{field}.type"] || 'application/octet-stream' ) end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb new file mode 100644 index 00000000000..322624c6092 --- /dev/null +++ b/lib/api/helpers/common_helpers.rb @@ -0,0 +1,13 @@ +module API + module Helpers + module CommonHelpers + def convert_parameters_from_legacy_format(params) + params.tap do |params| + if params[:assignee_id].present? + params[:assignee_ids] = [params.delete(:assignee_id)] + end + end + end + end + end +end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 718f936a1fc..264df7271a3 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -1,48 +1,14 @@ module API module Helpers module InternalHelpers - # Project paths may be any of the following: - # * /repository/storage/path/namespace/project - # * /namespace/project - # * namespace/project - # - # In addition, they may have a '.git' extension and multiple namespaces - # - # Transform all these cases to 'namespace/project' - def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values) - project_path = project_path.sub(/\.git\z/, '') - - storages.each do |storage| - storage_path = File.expand_path(storage['path']) - - if project_path.start_with?(storage_path) - project_path = project_path.sub(storage_path, '') - break - end - end - - project_path.sub(/\A\//, '') - end - - def project_path - @project_path ||= clean_project_path(params[:project]) - end - def wiki? - @wiki ||= project_path.end_with?('.wiki') && - !Project.find_by_full_path(project_path) + set_project unless defined?(@wiki) + @wiki end def project - @project ||= begin - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - Project.find_by_full_path(project_path) - end + set_project unless defined?(@project) + @project end def ssh_authentication_abilities @@ -66,6 +32,16 @@ module API ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) end + + private + + def set_project + if params[:gl_repository] + @project, @wiki = Gitlab::GlRepository.parse(params[:gl_repository]) + else + @project, @wiki = Gitlab::RepoPath.parse(params[:project]) + end + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 5b48ee8665f..96aaaf868ea 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -42,6 +42,10 @@ module API if access_status.status log_user_activity(actor) + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = @@ -86,7 +90,7 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION, + gitlab_rev: Gitlab::REVISION } end @@ -134,13 +138,11 @@ module API return unless Gitlab::GitalyClient.enabled? - relative_path = Gitlab::RepoPath.strip_storage_path(params[:repo_path]) - project = Project.find_by_full_path(relative_path.sub(/\.(git|wiki)\z/, '')) - begin - Gitlab::GitalyClient::Notifications.new(project.repository).post_receive + repository = wiki? ? project.wiki.repository : project.repository + Gitlab::GitalyClient::Notifications.new(repository.raw_repository).post_receive rescue GRPC::Unavailable => e - render_api_error(e, 500) + render_api_error!(e, 500) end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 244725bb292..78db960ae28 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -32,7 +32,8 @@ module API params :issue_params_ce do optional :description, type: String, desc: 'The description of an issue' - optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' + optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' @@ -135,6 +136,8 @@ module API issue_params = declared_params(include_missing: false) + issue_params = convert_parameters_from_legacy_format(issue_params) + issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute @@ -159,7 +162,7 @@ module API desc: 'Date time when the issue was updated. Available only for admins and project owners.' optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' use :issue_params - at_least_one_of :title, :description, :assignee_id, :milestone_id, + at_least_one_of :title, :description, :assignee_ids, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do @@ -173,6 +176,8 @@ module API update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) + issue = ::Issues::UpdateService.new(user_project, current_user, update_params).execute(issue) @@ -219,6 +224,21 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + desc 'List merge requests closing issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/closed_by' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id) + merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids) + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end end end end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 288b03d940c..0223957fde1 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -132,6 +132,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) build.cancel @@ -148,6 +149,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -165,6 +167,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -181,6 +184,7 @@ module API authorize_update_builds! build = get_build!(params[:job_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -201,6 +205,7 @@ module API build = get_build!(params[:job_id]) + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -211,12 +216,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index cb7aec47cf0..710deba5ae3 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -20,6 +20,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -33,6 +35,17 @@ module API end end + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -57,23 +70,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? - - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end @@ -197,14 +202,15 @@ module API end put ':id/merge_requests/:merge_request_iid/merge' do merge_request = find_project_merge_request(params[:merge_request_iid]) + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! unless merge_request.mergeable_state? + not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) if params[:sha] && merge_request.diff_head_sha != params[:sha] render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) @@ -215,7 +221,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 754c3d85a04..9117704aa46 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -14,13 +14,23 @@ module API end params do use :pagination - optional :scope, type: String, values: %w(running branches tags), - desc: 'Either running, branches, or tags' + optional :scope, type: String, values: %w[running pending finished branches tags], + desc: 'The scope of pipelines' + optional :status, type: String, values: HasStatus::AVAILABLE_STATUSES, + desc: 'The status of pipelines' + optional :ref, type: String, desc: 'The ref of pipelines' + optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' + optional :name, type: String, desc: 'The name of the user who triggered pipelines' + optional :username, type: String, desc: 'The username of the user who triggered pipelines' + optional :order_by, type: String, values: PipelinesFinder::ALLOWED_INDEXED_COLUMNS, default: 'id', + desc: 'Order pipelines' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Sort pipelines' end get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index cfee38a9baf..98bc9c28527 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -17,8 +17,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index db4b31b55bc..ed5004e8d1a 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -26,6 +26,10 @@ module API params :optional_params do use :optional_params_ce end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end end resource :projects do @@ -56,10 +60,6 @@ module API optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end - params :statistics_params do - optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' - end - params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' @@ -69,7 +69,7 @@ module API options = options.reverse_merge( with: Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -85,6 +85,7 @@ module API end params do use :collection_params + use :statistics_params end get do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails @@ -151,10 +152,13 @@ module API desc 'Get a single project' do success Entities::ProjectWithAccess end + params do + use :statistics_params + end get ":id" do entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails present user_project, with: entity, current_user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end desc 'Get events for a single project' do @@ -222,7 +226,7 @@ module API :shared_runners_enabled, :snippets_enabled, :visibility, - :wiki_enabled, + :wiki_enabled ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' @@ -381,7 +385,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/services.rb b/lib/api/services.rb index 23ef62c2258..cb07df9e249 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -356,7 +356,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -559,7 +559,7 @@ module API SlackService, MattermostService, MicrosoftTeamsService, - TeamcityService, + TeamcityService ] if Rails.env.development? @@ -577,7 +577,7 @@ module API service_classes += [ MockCiService, MockDeploymentService, - MockMonitoringService, + MockMonitoringService ] end diff --git a/lib/api/session.rb b/lib/api/session.rb index 002ffd1d154..016415c3023 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserWithPrivateToken + success Entities::UserWithPrivateDetails end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateToken + present user, with: Entities::UserWithPrivateDetails end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index d01c7f2703b..82f513c984e 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -58,6 +58,7 @@ module API :restricted_visibility_levels, :send_user_confirmation_email, :sentry_enabled, + :clientside_sentry_enabled, :session_expire_delay, :shared_runners_enabled, :sidekiq_throttling_enabled, @@ -138,6 +139,10 @@ module API given sentry_enabled: ->(val) { val } do requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' end + optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/' + given clientside_sentry_enabled: ->(val) { val } do + requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name' + end optional :repository_storage, type: String, desc: 'Storage paths for new projects' optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." optional :koding_enabled, type: Boolean, desc: 'Enable Koding' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b93fdc62808..53f5953a8fb 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index dbe54d3cd31..91567909998 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -5,7 +5,7 @@ module API subscribable_types = { 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/users.rb b/lib/api/users.rb index 46f221f68fe..40acaebf670 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -433,7 +433,7 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 4dd03cdf24b..21935922414 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -134,6 +134,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) build.cancel @@ -150,6 +151,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) @@ -167,6 +169,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return forbidden!('Build is not erasable!') unless build.erasable? build.erase(erased_by: current_user) @@ -183,6 +186,7 @@ module API authorize_update_builds! build = get_build!(params[:build_id]) + authorize!(:update_build, build) return not_found!(build) unless build.artifacts? build.keep_artifacts! @@ -202,7 +206,7 @@ module API authorize_read_builds! build = get_build!(params[:build_id]) - + authorize!(:update_build, build) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) @@ -213,12 +217,12 @@ module API end helpers do - def get_build(id) + def find_build(id) user_project.builds.find_by(id: id.to_i) end def get_build!(id) - get_build(id) || not_found! + find_build(id) || not_found! end def present_artifacts!(artifacts_file) diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 832b4bdeb4f..56a9b019f1b 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -69,7 +69,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -129,7 +131,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -248,6 +252,13 @@ module API expose :project_id, :issues_events, :merge_requests_events expose :note_events, :build_events, :pipeline_events, :wiki_page_events end + + class Issue < ::API::Entities::Issue + unexpose :assignees + expose :assignee do |issue, options| + ::API::Entities::UserBasic.represent(issue.assignees.first, options) + end + end end end end diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index 63d464b926b..6187445fc8d 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -20,7 +20,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -45,7 +45,7 @@ module API groups = if current_user.admin Group.all elsif params[:all_available] - GroupsFinder.new.execute(current_user) + GroupsFinder.new(current_user).execute else current_user.groups end diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index 715083fc4f8..cb371fdbab8 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -8,6 +8,7 @@ module API helpers do def find_issues(args = {}) args = params.merge(args) + args = convert_parameters_from_legacy_format(args) args.delete(:id) args[:milestone_title] = args.delete(:milestone) @@ -51,7 +52,7 @@ module API resource :issues do desc "Get currently authenticated user's issues" do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -61,7 +62,7 @@ module API get do issues = find_issues(scope: 'authored') - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -70,7 +71,7 @@ module API end resource :groups, requirements: { id: %r{[^/]+} } do desc 'Get a list of group issues' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -82,7 +83,7 @@ module API issues = find_issues(group_id: group.id, match_all_labels: true) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user end end @@ -94,7 +95,7 @@ module API desc 'Get a list of project issues' do detail 'iid filter is deprecated have been removed on V4' - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -107,22 +108,22 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Get a single project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' end get ":id/issues/:issue_id" do issue = find_project_issue(params[:issue_id]) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end desc 'Create a new project issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :title, type: String, desc: 'The title of an issue' @@ -140,6 +141,7 @@ module API issue_params = declared_params(include_missing: false) issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions)) + issue_params = convert_parameters_from_legacy_format(issue_params) issue = ::Issues::CreateService.new(user_project, current_user, @@ -147,14 +149,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Update an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -176,6 +178,7 @@ module API end update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = convert_parameters_from_legacy_format(update_params) issue = ::Issues::UpdateService.new(user_project, current_user, @@ -184,14 +187,14 @@ module API render_spam_error! if issue.spam? if issue.valid? - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end desc 'Move an existing issue' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :issue_id, type: Integer, desc: 'The ID of a project issue' @@ -206,7 +209,7 @@ module API begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + present issue, with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 3077240e650..b6b7254ae29 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -23,6 +23,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -32,7 +34,7 @@ module API if project.has_external_issue_tracker? ::API::Entities::ExternalIssue else - ::API::Entities::Issue + ::API::V3::Entities::Issue end end diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index be90cec4afc..4c7061d4939 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -39,7 +39,7 @@ module API end desc 'Get all issues for a single project milestone' do - success ::API::Entities::Issue + success ::API::V3::Entities::Issue end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -56,7 +56,7 @@ module API } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: ::API::V3::Entities::Issue, current_user: current_user, project: user_project end end end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index 82827249244..c48cbd2b765 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -21,7 +21,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute present paginate(pipelines), with: ::API::Entities::Pipeline end end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index fc065a22d74..c41fee32610 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -18,8 +18,7 @@ module API end def snippets_for_current_user - finder_params = { filter: :by_project, project: user_project } - SnippetsFinder.new.execute(current_user, finder_params) + SnippetsFinder.new(current_user, project: user_project).execute end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index ba9748ada59..164612cb8dd 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -88,7 +88,7 @@ module API options = options.reverse_merge( with: ::API::V3::Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -452,7 +452,7 @@ module API requires :file, type: File, desc: 'The file to be uploaded' end post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + UploadService.new(user_project, params[:file]).execute end desc 'Get the users list of a project' do diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 61629a04174..118c6df6549 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -377,7 +377,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 07dac7e9904..0762fc02d70 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -8,11 +8,11 @@ module API resource :snippets do helpers do def snippets_for_current_user - SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + SnippetsFinder.new(current_user, author: current_user).execute end def public_snippets - SnippetsFinder.new.execute(current_user, filter: :public) + SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute end end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb index 068750ec077..690768db82f 100644 --- a/lib/api/v3/subscriptions.rb +++ b/lib/api/v3/subscriptions.rb @@ -7,7 +7,7 @@ module API 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4016ac76348..d97e5d98229 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -80,16 +80,32 @@ module Backup 'port' => '--port', 'socket' => '--socket', 'username' => '--user', - 'encoding' => '--default-character-set' + 'encoding' => '--default-character-set', + # SSL + 'sslkey' => '--ssl-key', + 'sslcert' => '--ssl-cert', + 'sslca' => '--ssl-ca', + 'sslcapath' => '--ssl-capath', + 'sslcipher' => '--ssl-cipher' } args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + args = { + 'username' => 'PGUSER', + 'host' => 'PGHOST', + 'port' => 'PGPORT', + 'password' => 'PGPASSWORD', + # SSL + 'sslmode' => 'PGSSLMODE', + 'sslkey' => 'PGSSLKEY', + 'sslcert' => 'PGSSLCERT', + 'sslrootcert' => 'PGSSLROOTCERT', + 'sslcrl' => 'PGSSLCRL', + 'sslcompression' => 'PGSSLCOMPRESSION' + } + args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] } end def report_success(success) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..330cd963626 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -15,11 +15,10 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(Gitlab.config.backup.path) do - File.open("#{Gitlab.config.backup.path}/backup_information.yml", - "w+") do |file| + Dir.chdir(backup_path) do + File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/, '') end @@ -64,9 +63,9 @@ module Backup $progress.print "Deleting tmp directories ... " backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + next unless File.exist?(File.join(backup_path, dir)) - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(backup_path, dir)) $progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) @@ -83,8 +82,8 @@ module Backup if keep_time > 0 removed = 0 - Dir.chdir(Gitlab.config.backup.path) do - Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + Dir.chdir(backup_path) do + backup_file_list.each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -107,18 +106,14 @@ module Backup end def unpack - Dir.chdir(Gitlab.config.backup.path) + Dir.chdir(backup_path) # check for existing backups in the backup dir - file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") - - if file_list.count == 0 - $progress.puts "No backups found in #{Gitlab.config.backup.path}" + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - end - - if file_list.count > 1 && ENV["BACKUP"].nil? + elsif backup_file_list.many? && ENV["BACKUP"].nil? $progress.puts 'Found more than one backup, please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 @@ -127,7 +122,7 @@ module Backup tar_file = if ENV['BACKUP'].present? "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" else - file_list.first + backup_file_list.first end unless File.exist?(tar_file) @@ -169,6 +164,14 @@ module Backup private + def backup_path + Gitlab.config.backup.path + end + + def backup_file_list + @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") + end + def connect_to_remote_directory(connection_settings) connection = ::Fog::Storage.new(connection_settings) diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d6138816e70..6255a611dbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -53,7 +53,10 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): + (?=[^[:alnum:]:]|$)/x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index d67d466bce8..7d15a0f6d44 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -2,16 +2,17 @@ module Banzai module Filter # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter + SCHEMES = ['http', 'https', nil].freeze + def call links.each do |node| - href = href_to_lowercase_scheme(node["href"].to_s) + uri = uri(node['href'].to_s) + next unless uri - unless node["href"].to_s == href - node.set_attribute('href', href) - end + node.set_attribute('href', uri.to_s) - if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) - node.set_attribute('rel', 'nofollow noreferrer') + if SCHEMES.include?(uri.scheme) && external_url?(uri) + node.set_attribute('rel', 'nofollow noreferrer noopener') node.set_attribute('target', '_blank') end end @@ -21,27 +22,26 @@ module Banzai private + def uri(href) + URI.parse(href) + rescue URI::InvalidURIError + nil + end + def links query = 'descendant-or-self::a[@href and not(@href = "")]' doc.xpath(query) end - def href_to_lowercase_scheme(href) - scheme_match = href.match(/\A(\w+):\/\//) - - if scheme_match - scheme_match.to_s.downcase + scheme_match.post_match - else - href - end - end + def external_url?(uri) + # Relative URLs miss a hostname + return false unless uri.hostname - def external_url?(url) - !url.start_with?(internal_url) + uri.hostname != internal_url.hostname end def internal_url - @internal_url ||= Gitlab.config.gitlab.url + @internal_url ||= URI.parse(Gitlab.config.gitlab.url) end end end diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index c5ce360e172..cbabf9156de 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,9 +28,13 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issue_parser.issues_for_nodes(nodes).merge( + issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( merge_request_parser.merge_requests_for_nodes(nodes) ) + + # The project for the issue/MR might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes.select { |node, issuable| issuable.project } end end end diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb new file mode 100644 index 00000000000..c56d908009f --- /dev/null +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -0,0 +1,13 @@ +module Banzai + module Pipeline + class MarkupPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::PlantumlFilter + ] + end + end + end +end diff --git a/lib/banzai/reference_parser/issue_parser.rb b/lib/banzai/reference_parser/issue_parser.rb index e02b360924a..89ec715ddf6 100644 --- a/lib/banzai/reference_parser/issue_parser.rb +++ b/lib/banzai/reference_parser/issue_parser.rb @@ -28,7 +28,7 @@ module Banzai nodes, Issue.all.includes( :author, - :assignee, + :assignees, { # These associations are primarily used for checking permissions. # Eager loading these ensures we don't end up running dozens of diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index 84a28b33d7c..8b0662749fd 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -33,7 +33,8 @@ module Banzai { namespace: :owner }, { group: [:owners, :group_members] }, :invited_groups, - :project_members + :project_members, + :project_feature ] }), self.class.data_attribute diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b439b0ee29b..55402101e43 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -20,7 +20,7 @@ module Ci italic: 0x02, underline: 0x04, conceal: 0x08, - cross: 0x10, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 15a461a16dd..b06474cda7f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -70,7 +70,7 @@ module Ci cache: job[:cache], dependencies: job[:dependencies], after_script: job[:after_script], - environment: job[:environment], + environment: job[:environment] }.compact } end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index bae4db1ca4d..5f379756c11 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,16 +2,8 @@ class GroupUrlConstrainer def matches?(request) id = request.params[:id] - return false unless valid?(id) + return false unless DynamicPathValidator.valid?(id) - Group.find_by_full_path(id).present? - end - - private - - def valid?(id) - id.split('/').all? do |namespace| - NamespaceValidator.valid?(namespace) - end + Group.find_by_full_path(id, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index a10b4657d7d..6f542f63f98 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,10 +4,8 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = namespace_path + '/' + project_path - unless ProjectPathValidator.valid?(project_path) - return false - end + return false unless DynamicPathValidator.valid?(full_path) - Project.find_by_full_path(full_path).present? + Project.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 9ab5bcb12ff..28159dc0dec 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,5 +1,5 @@ class UserUrlConstrainer def matches?(request) - User.find_by_username(request.params[:username]).present? + User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present? end end diff --git a/lib/github/client.rb b/lib/github/client.rb new file mode 100644 index 00000000000..e65d908d232 --- /dev/null +++ b/lib/github/client.rb @@ -0,0 +1,23 @@ +module Github + class Client + attr_reader :connection, :rate_limit + + def initialize(options) + @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, 60) + faraday.options.timeout = options.fetch(:timeout, 60) + faraday.authorization 'token', options.fetch(:token) + faraday.adapter :net_http + end + + @rate_limit = RateLimit.new(connection) + end + + def get(url, query = {}) + exceed, reset_in = rate_limit.get + sleep reset_in if exceed + + Github::Response.new(connection.get(url, query)) + end + end +end diff --git a/lib/github/collection.rb b/lib/github/collection.rb new file mode 100644 index 00000000000..014b2038c4b --- /dev/null +++ b/lib/github/collection.rb @@ -0,0 +1,29 @@ +module Github + class Collection + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch(url, query = {}) + return [] if url.blank? + + Enumerator.new do |yielder| + loop do + response = client.get(url, query) + response.body.each { |item| yielder << item } + + raise StopIteration unless response.rels.key?(:next) + url = response.rels[:next] + end + end.lazy + end + + private + + def client + @client ||= Github::Client.new(options) + end + end +end diff --git a/lib/github/error.rb b/lib/github/error.rb new file mode 100644 index 00000000000..66d7afaa787 --- /dev/null +++ b/lib/github/error.rb @@ -0,0 +1,3 @@ +module Github + RepositoryFetchError = Class.new(StandardError) +end diff --git a/lib/github/import.rb b/lib/github/import.rb new file mode 100644 index 00000000000..06beb607a3e --- /dev/null +++ b/lib/github/import.rb @@ -0,0 +1,409 @@ +require_relative 'error' +module Github + class Import + include Gitlab::ShellAdapter + + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Note < ::Note + self.table_name = 'notes' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose + + def initialize(project, options) + @project = project + @repository = project.repository + @repo = project.import_source + @options = options + @verbose = options.fetch(:verbose, false) + @cached = Hash.new { |hash, key| hash[key] = Hash.new } + @errors = [] + end + + # rubocop: disable Rails/Output + def execute + puts 'Fetching repository...'.color(:aqua) if verbose + fetch_repository + puts 'Fetching labels...'.color(:aqua) if verbose + fetch_labels + puts 'Fetching milestones...'.color(:aqua) if verbose + fetch_milestones + puts 'Fetching pull requests...'.color(:aqua) if verbose + fetch_pull_requests + puts 'Fetching issues...'.color(:aqua) if verbose + fetch_issues + puts 'Cloning wiki repository...'.color(:aqua) if verbose + fetch_wiki_repository + puts 'Expiring repository cache...'.color(:aqua) if verbose + expire_repository_cache + + true + rescue Github::RepositoryFetchError + false + ensure + keep_track_of_errors + end + + private + + def fetch_repository + begin + project.create_repository unless project.repository.exists? + project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.set_remote_as_mirror('github') + project.repository.fetch_remote('github', forced: true) + rescue Gitlab::Shell::Error => e + error(:project, "https://github.com/#{repo}.git", e.message) + raise Github::RepositoryFetchError + end + end + + def fetch_wiki_repository + wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git" + wiki_path = "#{project.path_with_namespace}.wiki" + + unless project.wiki.repository_exists? + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors(:wiki, wiki_url, e.message) + end + end + + def fetch_labels + url = "/repos/#{repo}/labels" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + begin + representation = Github::Representation::Label.new(raw) + + label = project.labels.find_or_create_by!(title: representation.title) do |label| + label.color = representation.color + end + + cached[:label_ids][label.title] = label.id + rescue => e + error(:label, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_milestones + url = "/repos/#{repo}/milestones" + + while url + response = Github::Client.new(options).get(url, state: :all) + + response.body.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + next if project.milestones.where(iid: milestone.iid).exists? + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_pull_requests + url = "/repos/#{repo}/pulls" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) + next unless merge_request.new_record? && pull_request.valid? + + begin + restore_branches(pull_request) + + author_id = user_id(pull_request.author, project.creator_id) + description = format_description(pull_request.description, pull_request.author) + + merge_request.attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: pull_request.source_project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: pull_request.target_project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + milestone_id: milestone_id(pull_request.milestone), + author_id: author_id, + assignee_id: user_id(pull_request.assignee), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + merge_request.save!(validate: false) + merge_request.merge_request_diffs.create + + # Fetch review comments + review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" + fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) + + # Fetch comments + comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" + fetch_comments(merge_request, :comment, comments_url) + rescue => e + error(:pull_request, pull_request.url, e.message) + ensure + clean_up_restored_branches(pull_request) + end + end + + url = response.rels[:next] + end + end + + def fetch_issues + url = "/repos/#{repo}/issues" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + next unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_ids = [user_id(representation.assignee)] + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) + end + end + rescue => e + error(:issue, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_comments(noteable, type, url, klass = Note) + while url + comments = Github::Client.new(options).get(url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + representation = Github::Representation::Comment.new(raw, options) + author_id = user_id(representation.author, project.creator_id) + + note = klass.new + note.project_id = project.id + note.noteable = noteable + note.note = format_description(representation.note, representation.author) + note.commit_id = representation.commit_id + note.line_code = representation.line_code + note.author_id = author_id + note.created_at = representation.created_at + note.updated_at = representation.updated_at + note.save!(validate: false) + rescue => e + error(type, representation.url, e.message) + end + end + end + + url = comments.rels[:next] + end + end + + def fetch_releases + url = "/repos/#{repo}/releases" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + representation = Github::Representation::Release.new(raw) + next unless representation.valid? + + release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) + next unless relese.new_record? + + begin + release.description = representation.description + release.created_at = representation.created_at + release.updated_at = representation.updated_at + release.save!(validate: false) + rescue => e + error(:release, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def restore_branches(pull_request) + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + end + + def restore_source_branch(pull_request) + repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def label_ids(labels) + labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + end + + def milestone_id(milestone) + return unless milestone.present? + + project.milestones.select(:id).find_by(iid: milestone.iid)&.id + end + + def user_id(user, fallback_id = nil) + return unless user.present? + return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) + + gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) + + cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? + cached[:user_ids][user.id] = gitlab_user_id || fallback_id + end + + def user_id_by_email(email) + return nil unless email + + ::User.find_by_any_email(email)&.id + end + + def user_id_by_external_uid(id) + return nil unless id + + ::User.select(:id) + .joins(:identities) + .merge(::Identity.where(provider: :github, extern_uid: id)) + .first&.id + end + + def format_description(body, author) + return body if cached[:gitlab_user_ids][author.id] + + "*Created by: #{author.username}*\n\n#{body}" + end + + def expire_repository_cache + repository.expire_content_cache + end + + def keep_track_of_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def error(type, url, message) + errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } + end + end +end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb new file mode 100644 index 00000000000..884693d093c --- /dev/null +++ b/lib/github/rate_limit.rb @@ -0,0 +1,27 @@ +module Github + class RateLimit + SAFE_REMAINING_REQUESTS = 100 + SAFE_RESET_TIME = 500 + RATE_LIMIT_URL = '/rate_limit'.freeze + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def get + response = connection.get(RATE_LIMIT_URL) + + # GitHub Rate Limit API returns 404 when the rate limit is disabled + return false unless response.status != 404 + + body = Oj.load(response.body, class_cache: false, mode: :compat) + remaining = body.dig('rate', 'remaining').to_i + reset_in = body.dig('rate', 'reset').to_i + exceed = remaining <= SAFE_REMAINING_REQUESTS + + [exceed, reset_in] + end + end +end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb new file mode 100644 index 00000000000..c1c9448f305 --- /dev/null +++ b/lib/github/repositories.rb @@ -0,0 +1,19 @@ +module Github + class Repositories + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch + Collection.new(options).fetch(repos_url) + end + + private + + def repos_url + '/user/repos' + end + end +end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb new file mode 100644 index 00000000000..f26bdbdd546 --- /dev/null +++ b/lib/github/representation/base.rb @@ -0,0 +1,30 @@ +module Github + module Representation + class Base + def initialize(raw, options = {}) + @raw = raw + @options = options + end + + def id + raw['id'] + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + private + + attr_reader :raw, :options + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb new file mode 100644 index 00000000000..d1dac6944f0 --- /dev/null +++ b/lib/github/representation/branch.rb @@ -0,0 +1,51 @@ +module Github + module Representation + class Branch < Representation::Base + attr_reader :repository + + def user + raw.dig('user', 'login') || 'unknown' + end + + def repo + return @repo if defined?(@repo) + + @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + end + + def ref + raw['ref'] + end + + def sha + raw['sha'] + end + + def short_sha + Commit.truncate_sha(sha) + end + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + private + + def branch_exists? + repository.branch_exists?(ref) + end + + def commit_exists? + repository.branch_names_contains(sha).include?(ref) + end + + def repository + @repository ||= options.fetch(:repository) + end + end + end +end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb new file mode 100644 index 00000000000..1b5be91461b --- /dev/null +++ b/lib/github/representation/comment.rb @@ -0,0 +1,42 @@ +module Github + module Representation + class Comment < Representation::Base + def note + raw['body'] || '' + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def commit_id + raw['commit_id'] + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + private + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw['diff_hunk'] + end + + def file_path + raw['path'] + end + end + end +end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb new file mode 100644 index 00000000000..9713b82615d --- /dev/null +++ b/lib/github/representation/issuable.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Issuable < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee'], options) + end + + def assigned? + raw['assignee'].present? + end + end + end +end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb new file mode 100644 index 00000000000..df3540a6e6c --- /dev/null +++ b/lib/github/representation/issue.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Issue < Representation::Issuable + def labels + raw['labels'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'opened' + end + + def has_comments? + raw['comments'] > 0 + end + + def has_labels? + labels.count > 0 + end + + def pull_request? + raw['pull_request'].present? + end + end + end +end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb new file mode 100644 index 00000000000..60aa51f9569 --- /dev/null +++ b/lib/github/representation/label.rb @@ -0,0 +1,13 @@ +module Github + module Representation + class Label < Representation::Base + def color + "##{raw['color']}" + end + + def title + raw['name'] + end + end + end +end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb new file mode 100644 index 00000000000..917e6394ad4 --- /dev/null +++ b/lib/github/representation/milestone.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Milestone < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['description'] + end + + def due_date + raw['due_on'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'active' + end + end + end +end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb new file mode 100644 index 00000000000..ac9c8283b4b --- /dev/null +++ b/lib/github/representation/pull_request.rb @@ -0,0 +1,78 @@ +module Github + module Representation + class PullRequest < Representation::Issuable + attr_reader :project + + delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + + def source_project + project + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def target_project + project + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def state + return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? + return 'closed' if raw['state'] == 'closed' + + 'opened' + end + + def opened? + state == 'opened' + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + private + + def project + @project ||= options.fetch(:project) + end + + def source_branch + @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" + end + + def target_branch + @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + end + end +end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb new file mode 100644 index 00000000000..e7e4b428c1a --- /dev/null +++ b/lib/github/representation/release.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Release < Representation::Base + def description + raw['body'] + end + + def tag + raw['tag_name'] + end + + def valid? + !raw['draft'] + end + end + end +end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb new file mode 100644 index 00000000000..6938aa7db05 --- /dev/null +++ b/lib/github/representation/repo.rb @@ -0,0 +1,6 @@ +module Github + module Representation + class Repo < Representation::Base + end + end +end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb new file mode 100644 index 00000000000..18591380e25 --- /dev/null +++ b/lib/github/representation/user.rb @@ -0,0 +1,15 @@ +module Github + module Representation + class User < Representation::Base + def email + return @email if defined?(@email) + + @email = Github::User.new(username, options).get.fetch('email', nil) + end + + def username + raw['login'] + end + end + end +end diff --git a/lib/github/response.rb b/lib/github/response.rb new file mode 100644 index 00000000000..761c524b553 --- /dev/null +++ b/lib/github/response.rb @@ -0,0 +1,25 @@ +module Github + class Response + attr_reader :raw, :headers, :status + + def initialize(response) + @raw = response + @headers = response.headers + @status = response.status + end + + def body + Oj.load(raw.body, class_cache: false, mode: :compat) + end + + def rels + links = headers['Link'].to_s.split(', ').map do |link| + href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures + + [name.to_sym, href] + end + + Hash[*links.flatten] + end + end +end diff --git a/lib/github/user.rb b/lib/github/user.rb new file mode 100644 index 00000000000..f88a29e590b --- /dev/null +++ b/lib/github/user.rb @@ -0,0 +1,24 @@ +module Github + class User + attr_reader :username, :options + + def initialize(username, options) + @username = username + @options = options + end + + def get + client.get(user_url).body + end + + private + + def client + @client ||= Github::Client.new(options) + end + + def user_url + "/users/#{username}" + end + end +end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 8c28009b9c6..4714ab18cc1 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -32,7 +32,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Master" => MASTER, + "Master" => MASTER } end @@ -47,7 +47,7 @@ module Gitlab guest: GUEST, reporter: REPORTER, developer: DEVELOPER, - master: MASTER, + master: MASTER } end @@ -60,7 +60,7 @@ module Gitlab "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, + "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL } end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d575367d81a..96d38f6daa0 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -14,30 +14,18 @@ module Gitlab # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref - # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter # - def self.render(input, context, asciidoc_opts = {}) - asciidoc_opts.reverse_merge!( - safe: :secure, - backend: :gitlab_html5, - attributes: [] - ) - asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + def self.render(input, context) + asciidoc_opts = { safe: :secure, + backend: :gitlab_html5, + attributes: DEFAULT_ADOC_ATTRS } + + context[:pipeline] = :markup plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) - - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..ea918b23a63 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -108,7 +108,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b358f2efa4f..4fc9a075edc 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -15,18 +15,51 @@ module Gitlab end end + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + def self.update_for_pipeline(pipeline) - new(pipeline.project, - sha: pipeline.sha, - status: pipeline.status, - ref: pipeline.ref).store_in_cache_if_needed + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed end - def initialize(project, sha: nil, status: nil, ref: nil) + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) @project = project - @sha = sha - @ref = ref - @status = status + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache end def has_status? @@ -85,6 +118,8 @@ module Gitlab end def has_cache? + return self.loaded unless self.loaded.nil? + Gitlab::Redis.with do |redis| redis.exists(cache_key) end @@ -95,7 +130,7 @@ module Gitlab end def cache_key - "projects/#{project.id}/build_status" + self.class.cache_key_for_project(project) end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index f34ed0f4cf2..3e0c30c33b7 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -5,7 +5,7 @@ module Gitlab Gitlab::ChatCommands::IssueShow, Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy, + Gitlab::ChatCommands::Deploy ].freeze def execute diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/chat_commands/presenters/issue_base.rb index 054f7f4be0c..25bc82994ba 100644 --- a/lib/gitlab/chat_commands/presenters/issue_base.rb +++ b/lib/gitlab/chat_commands/presenters/issue_base.rb @@ -22,7 +22,7 @@ module Gitlab [ { title: "Assignee", - value: @resource.assignee ? @resource.assignee.name : "_None_", + value: @resource.assignees.any? ? @resource.assignees.first.name : "_None_", short: true }, { diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 8793b20aa35..c984eb20606 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,7 +1,6 @@ module Gitlab module Checks class ChangeAccess - # protocol is currently used only in EE attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( @@ -18,7 +17,9 @@ module Gitlab end def exec - error = push_checks || tag_checks || protected_branch_checks + return GitAccessStatus.new(true) if skip_authorization + + error = push_checks || branch_checks || tag_checks if error GitAccessStatus.new(false, error) @@ -29,35 +30,59 @@ module Gitlab protected - def protected_branch_checks - return if skip_authorization + def push_checks + if user_access.cannot_do_action?(:push_code) + "You are not allowed to push code to this project." + end + end + + def branch_checks return unless @branch_name + + if deletion? && @branch_name == project.default_branch + return "The default branch of a project cannot be deleted." + end + + protected_branch_checks + end + + def protected_branch_checks return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif deletion? - return "You are not allowed to delete protected branches from this project." end + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + unless user_access.can_delete_branch?(@branch_name) + return 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.' + end + + unless protocol == 'web' + 'You can only delete protected branches using the web interface.' + end + end + + def protected_branch_push_checks if matching_merge_request? - if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) "You are not allowed to merge code into protected branches on this project." end else - if user_access.can_push_to_branch?(@branch_name) - return - else + unless user_access.can_push_to_branch?(@branch_name) "You are not allowed to push code to protected branches on this project." end end end def tag_checks - return if skip_authorization - return unless @tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) @@ -68,7 +93,8 @@ module Gitlab end def protected_tag_checks - return unless tag_protected? + return unless ProtectedTag.protected?(project, @tag_name) + return "Protected tags cannot be updated." if update? return "Protected tags cannot be deleted." if deletion? @@ -77,18 +103,6 @@ module Gitlab end end - def tag_protected? - ProtectedTag.protected?(project, @tag_name) - end - - def push_checks - return if skip_authorization - - if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." - end - end - private def tag_exists? diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 6f799c2f031..2e073334abc 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -37,6 +37,12 @@ module Gitlab !directory? end + def blob + return unless file? + + @blob ||= Blob.decorate(::Ci::ArtifactBlob.new(self), nil) + end + def has_parent? nodes > 0 end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index a3cc350ef22..551483d0aaa 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,12 +6,12 @@ module Gitlab def initialize(cron, cron_timezone = 'UTC') @cron = cron - @cron_timezone = cron_timezone + @cron_timezone = ActiveSupport::TimeZone.find_tzinfo(cron_timezone).name end def next_time_from(time) @cron_line ||= try_parse_cron(@cron, @cron_timezone) - @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present? + @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? end def cron_valid? @@ -24,8 +24,23 @@ module Gitlab private + # NOTE: + # cron_timezone can only accept timezones listed in TZInfo::Timezone. + # Aliases of Timezones from ActiveSupport::TimeZone are NOT accepted, + # because Rufus::Scheduler only supports TZInfo::Timezone. + # + # For example, those codes have the same effect. + # Time.zone = 'Pacific Time (US & Canada)' (ActiveSupport::TimeZone) + # Time.zone = 'America/Los_Angeles' (TZInfo::Timezone) + # + # However, try_parse_cron only accepts the latter format. + # try_parse_cron('* * * * *', 'Pacific Time (US & Canada)') -> Doesn't work + # try_parse_cron('* * * * *', 'America/Los_Angeles') -> Works + # If you want to know more, please take a look + # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/values/time_zone.rb def try_parse_cron(cron, cron_timezone) - Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line = Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + cron_line if cron_line.is_a?(Rufus::Scheduler::CronLine) rescue # noop end diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb new file mode 100644 index 00000000000..45fd0d4aa07 --- /dev/null +++ b/lib/gitlab/ci/status/build/action.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Action < Status::Extended + def label + if has_action? + @status.label + else + "#{@status.label} (not allowed)" + end + end + + def self.matches?(build, user) + build.action? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 67bbc3c4849..57b533bad99 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Cancelable < SimpleDelegator - include Status::Extended - + class Cancelable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 38ac6edc9f1..c852d607373 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -8,7 +8,8 @@ module Gitlab Status::Build::Retryable], [Status::Build::FailedAllowed, Status::Build::Play, - Status::Build::Stop]] + Status::Build::Stop], + [Status::Build::Action]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index 807afe24bd5..e42d3574357 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class FailedAllowed < SimpleDelegator - include Status::Extended - + class FailedAllowed < Status::Extended def label 'failed (allowed to fail)' end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 3495b8d0448..c6139f1b716 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Play < SimpleDelegator - include Status::Extended - + class Play < Status::Extended def label 'manual play action' end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 6b362af7634..505f80848b2 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Retryable < SimpleDelegator - include Status::Extended - + class Retryable < Status::Extended def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index e8530f2aaae..0b5199e5483 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Build - class Stop < SimpleDelegator - include Status::Extended - + class Stop < Status::Extended def label 'manual stop action' end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index d367c9bda69..1e8101f8949 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,13 +1,13 @@ module Gitlab module Ci module Status - module Extended - extend ActiveSupport::Concern + class Extended < SimpleDelegator + def initialize(status) + super(@status = status) + end - class_methods do - def matches?(_subject, _user) - raise NotImplementedError - end + def self.matches?(_subject, _user) + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb new file mode 100644 index 00000000000..cfd4329a923 --- /dev/null +++ b/lib/gitlab/ci/status/group/common.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Group + module Common + def has_details? + false + end + + def details_path + nil + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb new file mode 100644 index 00000000000..d118116cfc3 --- /dev/null +++ b/lib/gitlab/ci/status/group/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module Group + class Factory < Status::Factory + def self.common_helpers + Status::Group::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index a250c3fcb41..37dfe43fb62 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -2,9 +2,7 @@ module Gitlab module Ci module Status module Pipeline - class Blocked < SimpleDelegator - include Status::Extended - + class Blocked < Status::Extended def text 'blocked' end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index d4cdab6957a..df6e76b0151 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -5,9 +5,7 @@ module Gitlab # Extended status used when pipeline or stage passed conditionally. # This means that failed jobs that are allowed to fail were present. # - class SuccessWarning < SimpleDelegator - include Status::Extended - + class SuccessWarning < Status::Extended def text 'passed' end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 990b719ecfd..6e73361cad1 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -3,16 +3,33 @@ module Gitlab class FileCollection ConflictSideMissing = Class.new(StandardError) - attr_reader :merge_request, :our_commit, :their_commit + attr_reader :merge_request, :our_commit, :their_commit, :project - def initialize(merge_request) - @merge_request = merge_request - @our_commit = merge_request.source_branch_head.raw.raw_commit - @their_commit = merge_request.target_branch_head.raw.raw_commit - end + delegate :repository, to: :project + + class << self + # We can only write when getting the merge index from the source + # project, because we will write to that project. We don't use this all + # the time because this fetches a ref into the source project, which + # isn't needed for reading. + def for_resolution(merge_request) + project = merge_request.source_project + + new(merge_request, project).tap do |file_collection| + project. + repository. + with_repo_branch_commit(merge_request.target_project.repository, merge_request.target_branch) do + + yield file_collection + end + end + end - def repository - merge_request.project.repository + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + def read_only(merge_request) + new(merge_request, merge_request.target_project) + end end def merge_index @@ -55,6 +72,15 @@ Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branc #{conflict_filenames.join("\n")} EOM end + + private + + def initialize(merge_request, project) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + @project = project + end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 559e3939da6..cac31ea8cff 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -17,7 +17,7 @@ module Gitlab end def title - name.to_s.capitalize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def median diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 1e52b6614a1..5f9dc9a4303 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -13,12 +13,16 @@ module Gitlab :code end + def title + s_('CycleAnalyticsStage|Code') + end + def legend - "Related Merge Requests" + _("Related Merge Requests") end def description - "Time until first merge request" + _("Time until first merge request") end end end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 213994988a5..7b03811efb2 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -14,12 +14,16 @@ module Gitlab :issue end + def title + s_('CycleAnalyticsStage|Issue') + end + def legend - "Related Issues" + _("Related Issues") end def description - "Time before an issue gets scheduled" + _("Time before an issue gets scheduled") end end end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index bef3b95ff1b..1e11e84a9cb 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -7,7 +7,7 @@ module Gitlab test: :read_build, review: :read_merge_request, staging: :read_build, - production: :read_issue, + production: :read_issue }.freeze def self.get(*args) diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 45d51d30ccc..1a0afb56b4f 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -14,12 +14,16 @@ module Gitlab :plan end + def title + s_('CycleAnalyticsStage|Plan') + end + def legend - "Related Commits" + _("Related Commits") end def description - "Time before an issue starts implementation" + _("Time before an issue starts implementation") end end end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 9f387a02945..0fa8a65cb99 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -15,12 +15,16 @@ module Gitlab :production end + def title + s_('CycleAnalyticsStage|Production') + end + def legend - "Related Issues" + _("Related Issues") end def description - "From issue creation until deploy to production" + _("From issue creation until deploy to production") end def query diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index 4744be834de..cfbbdc43fd9 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -13,12 +13,16 @@ module Gitlab :review end + def title + s_('CycleAnalyticsStage|Review') + end + def legend - "Relative Merged Requests" + _("Related Merged Requests") end def description - "Time between merge request creation and merge/close" + _("Time between merge request creation and merge/close") end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index 3cdbe04fbaf..d5684bb9201 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -14,12 +14,16 @@ module Gitlab :staging end + def title + s_('CycleAnalyticsStage|Staging') + end + def legend - "Relative Deployed Builds" + _("Related Deployed Jobs") end def description - "From merge request merge until deploy to production" + _("From merge request merge until deploy to production") end end end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index 43fa3795e5c..a917ddccac7 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -8,7 +8,7 @@ module Gitlab end def title - self.class.name.demodulize + raise NotImplementedError.new("Expected #{self.name} to implement title") end def value diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 7b8faa4d854..bea78862757 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Commit < Base + def title + n_('Commit', 'Commits', value) + end + def value @value ||= count_commits end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 06032e9200e..099d798aac6 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -2,6 +2,10 @@ module Gitlab module CycleAnalytics module Summary class Deploy < Base + def title + n_('Deploy', 'Deploys', value) + end + def value @value ||= @project.deployments.where("created_at > ?", @from).count end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 008468f24b9..9bbf7a2685f 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -9,7 +9,7 @@ module Gitlab end def title - 'New Issue' + n_('New Issue', 'New Issues', value) end def value diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index e96943833bc..2b5f72bef89 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -13,12 +13,16 @@ module Gitlab :test end + def title + s_('CycleAnalyticsStage|Test') + end + def legend - "Relative Builds Trigger by Commits" + _("Related Jobs") end def description - "Total test time for all commits/merges" + _("Total test time for all commits/merges") end def stage_query diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index f78106f5b10..8e74e18a311 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -36,7 +36,7 @@ module Gitlab user: { id: user.try(:id), name: user.try(:name), - email: user.try(:email), + email: user.try(:email) }, commit: { @@ -49,7 +49,7 @@ module Gitlab status: commit.status, duration: commit.duration, started_at: commit.started_at, - finished_at: commit.finished_at, + finished_at: commit.finished_at }, repository: { @@ -60,7 +60,7 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - }, + } } data diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index d76aa38f741..e81d19a7a2e 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -11,6 +11,7 @@ module Gitlab # ref: String, # user_id: String, # user_name: String, + # user_username: String, # user_email: String # project_id: String, # repository: { @@ -41,7 +42,7 @@ module Gitlab type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data - data = { + { object_kind: type, event_name: type, before: oldrev, @@ -51,6 +52,7 @@ module Gitlab message: message, user_id: user.id, user_name: user.name, + user_username: user.username, user_email: user.email, user_avatar: user.avatar_url, project_id: project.id, @@ -61,16 +63,15 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } - - data end # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - build(project, user, commits.last.id, commits.first.id, ref, commits) + commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + + build(project, user, commits.last&.id, commits.first&.id, ref, commits) end def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb new file mode 100644 index 00000000000..b42dc052949 --- /dev/null +++ b/lib/gitlab/data_builder/repository.rb @@ -0,0 +1,35 @@ +module Gitlab + module DataBuilder + module Repository + extend self + + # Produce a hash of post-receive data + def update(project, user, changes, refs) + { + event_name: 'repository_update', + + user_id: user.id, + user_name: user.name, + user_email: user.email, + user_avatar: user.avatar_url, + + project_id: project.id, + project: project.hook_attrs, + + changes: changes, + + refs: refs + } + end + + # Produce a hash of partial data for a single change + def single_change(oldrev, newrev, ref) + { + before: oldrev, + after: newrev, + ref: ref + } + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6dabbe0264c..f3476dadec8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -278,6 +278,20 @@ module Gitlab raise 'rename_column_concurrently can not be run inside a transaction' end + old_col = column_for(table, old) + new_type = type || old_col.type + + add_column(table, new, new_type, + limit: old_col.limit, + null: old_col.null, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new, old_col.default) if old_col.default + trigger_name = rename_trigger_name(table, old, new) quoted_table = quote_table_name(table) quoted_old = quote_column_name(old) @@ -291,16 +305,6 @@ module Gitlab quoted_old, quoted_new) end - old_col = column_for(table, old) - new_type = type || old_col.type - - add_column(table, new, new_type, - limit: old_col.limit, - default: old_col.default, - null: old_col.null, - precision: old_col.precision, - scale: old_col.scale) - update_column_in_batches(table, new, Arel::Table.new(table)[old]) copy_indexes(table, old, new) @@ -498,6 +502,29 @@ module Gitlab columns(table).find { |column| column.name == name } end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..4fdcb682c2f --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,76 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..5397877b5d5 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,131 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%")) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..b9f4f3cff3c --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,72 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb new file mode 100644 index 00000000000..c45ae8feb2c --- /dev/null +++ b/lib/gitlab/dependency_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + LINKERS = [ + GemfileLinker + ].freeze + + def self.linker(blob_name) + LINKERS.find { |linker| linker.support?(blob_name) } + end + + def self.link(blob_name, plain_text, highlighted_text) + linker = linker(blob_name) + return highlighted_text unless linker + + linker.link(plain_text, highlighted_text) + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb new file mode 100644 index 00000000000..40a4ad11372 --- /dev/null +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -0,0 +1,103 @@ +module Gitlab + module DependencyLinker + class BaseLinker + def self.link(plain_text, highlighted_text) + new(plain_text, highlighted_text).link + end + + attr_accessor :plain_text, :highlighted_text + + def initialize(plain_text, highlighted_text) + @plain_text = plain_text + @highlighted_text = highlighted_text + end + + def link + link_dependencies + + highlighted_lines.join.html_safe + end + + private + + def package_url(name) + raise NotImplementedError + end + + def link_dependencies + raise NotImplementedError + end + + def package_link(name, url = package_url(name)) + return name unless url + + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="noopener noreferrer" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + end + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call("gem") + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call("gem", "specific_package") + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call("github", /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call("name") + # # Will link `package` in `self.name = "package"` + def link_method_call(method_names, value = nil, &url_proc) + value = + case value + when String + Regexp.escape(value) + when nil + /[^'"]+/ + else + value + end + + method_names = Array(method_names).map { |name| Regexp.escape(name) } + + regex = %r{ + #{Regexp.union(method_names)} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + + # Links package names based on regex. + # + # Example: + # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) + # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` + def link_regex(regex) + highlighted_lines.map!.with_index do |rich_line, i| + marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + + marker.mark(regex, group: :name) do |text, left:, right:| + url = block_given? ? yield(text) : package_url(text) + package_link(text, url) + end + end + end + + def plain_lines + @plain_lines ||= plain_text.lines + end + + def highlighted_lines + @highlighted_lines ||= highlighted_text.lines + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb new file mode 100644 index 00000000000..45be760d89e --- /dev/null +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -0,0 +1,31 @@ +module Gitlab + module DependencyLinker + class GemfileLinker < BaseLinker + def self.support?(blob_name) + blob_name == 'Gemfile' || blob_name == 'gems.rb' + end + + private + + def link_dependencies + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call("gem") + + # Link `github: "user/repo"` to https://github.com/user/repo + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/) do |name| + "https://github.com/#{name}" + end + + # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>https?://[^'"]+)['"]}) { |url| url } + + # Link `source "https://rubygems.org"` to https://rubygems.org + link_method_call("source", %r{https?://[^'"]+}) { |url| url } + end + + def package_url(name) + "https://rubygems.org/gems/#{name}" + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb new file mode 100644 index 00000000000..c2a2eb15931 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -0,0 +1,17 @@ +module Gitlab + module Diff + class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + }.freeze + + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + symbol = MARKDOWN_SYMBOLS[mode] + "{#{symbol}#{text}#{symbol}}" + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 736933b1c4b..919965100ae 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,137 +1,21 @@ module Gitlab module Diff - class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - }.freeze - - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) - end - - def mark(line_inline_diffs, mode: nil, markdown: false) - return rich_line unless line_inline_diffs - - marker_ranges = [] - line_inline_diffs.each do |inline_diff_range| - # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range].flatten - # Turn the array of character positions into ranges - marker_ranges.concat(collapse_ranges(inline_diff_positions)) - end - - offset = 0 - - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "<span class='#{html_class_names(marker_ranges, mode, index)}'>" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "</span>" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + class InlineDiffMarker < Gitlab::StringRangeMarker + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - - rich_line.html_safe end private - def html_class_names(marker_ranges, mode, index) + def html_class_names(left, right, mode) class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 + class_names << "left" if left + class_names << "right" if right class_names << mode if mode class_names.join(" ") end - - # Mapping of character positions in the raw line, to the rich (highlighted) line - def position_mapping - @position_mapping ||= begin - mapping = [] - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - rich_char = rich_line[rich_pos] - - # The raw and rich lines are the same except for HTML tags, - # so skip over any `<...>` segment - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - # multi-char HTML entities in the rich line correspond to a single character in the raw line - if rich_char == '&' - multichar_mapping = [rich_pos] - until rich_char == ';' - rich_pos += 1 - multichar_mapping << rich_pos - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = multichar_mapping - else - mapping[raw_pos] = rich_pos - end - - rich_pos += 1 - end - - mapping - end - end - - # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - # Inserts tags around the characters identified by the given range - def insert_around_range(text, range, before, after, offset = 0) - # Just to be sure - return offset if offset + range.end + 1 > text.length - - text.insert(offset + range.begin, before) - offset += before.length - - text.insert(offset + range.end + 1, after) - offset += after.length - - offset - end end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..c7542a8fabc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 32cece8316b..83440ae227d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -21,7 +21,7 @@ module Gitlab content_type: attachment.content_type } - link = ::Projects::UploadService.new(project, file).execute + link = UploadService.new(project, file).execute attachments << link if link ensure tmp.close! diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..a616a80e8f5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project&.full_path) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index c66b0435f3a..31579e94a87 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -28,6 +28,10 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project&.full_path) + end + private def author diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index df491f060bf..5894384da5d 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -19,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project&.full_path) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index bb4fdd1f1f4..0d6b08b5d29 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,9 +31,7 @@ module Gitlab raise UnknownIncomingEmail unless handler - Gitlab::Metrics.add_event(:receive_email, - project: handler.try(:project), - handler: handler.class.name) + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) handler.execute end @@ -60,9 +57,8 @@ module Gitlab end def key_from_additional_headers(mail) - references = ensure_references_array(mail.references) - - find_key_from_references(references) + find_key_from_references(mail) || + find_key_from_delivered_to_header(mail) end def ensure_references_array(references) @@ -73,15 +69,24 @@ module Gitlab # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app Gitlab::IncomingEmail.scan_fallback_references(references) + when nil + [] end end - def find_key_from_references(references) - references.find do |mail_id| + def find_key_from_references(mail) + ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end + + def find_key_from_delivered_to_header(mail) + Array(mail[:delivered_to]).find do |header| + key = Gitlab::IncomingEmail.key_from_address(header.value) + break key if key + end + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index a16d9fc2265..e3e36b35ce9 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -54,7 +54,7 @@ module Gitlab unicode_version: emoji_unicode_version(emoji_name) } - ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], data: data) + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], title: emoji_info['description'], data: data) end end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index f6e4f279c06..31a5b9d108b 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,32 +2,44 @@ module Gitlab module EtagCaching class Router Route = Struct.new(:regexp, :name) - - RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/rendered_title` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues rendered_title + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/commit/\S+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), 'commit_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/new\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), 'new_merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/\d+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), 'project_pipelines' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z), + 'project_pipeline' ) ].freeze diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index c9ca4cadd1c..f8b3d0b4965 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -13,7 +13,8 @@ module Gitlab gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - avatar: /\Alogo\.(png|jpg|gif)\z/ + avatar: /\Alogo\.(png|jpg|gif)\z/, + route_map: 'route-map.yml' }.freeze # Returns an Array of file types based on the given paths. diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb new file mode 100644 index 00000000000..093d9ed8092 --- /dev/null +++ b/lib/gitlab/file_finder.rb @@ -0,0 +1,32 @@ +# This class finds files in a repository by name and content +# the result is joined and sorted by file name +module Gitlab + class FileFinder + BATCH_SIZE = 100 + + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + end + + def find(query) + blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = Gitlab::ProjectSearchResults.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| + results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 222bcdcbf9c..3dcee681c72 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -122,15 +122,15 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - iid: bug['ixBug'], - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed', - created_at: date, - updated_at: DateTime.parse(bug['dtLastUpdated']) + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_ids: [assignee_id], + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 98fd4e78126..c1b31618e0d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -90,7 +90,7 @@ module Gitlab name: blob_entry[:name], data: '', path: path, - commit_id: sha, + commit_id: sha ) end end @@ -109,10 +109,6 @@ module Gitlab @binary.nil? ? super : @binary == true end - def empty? - !data || data == '' - end - def data encode! @data end @@ -132,6 +128,10 @@ module Gitlab encode! @name end + def truncated? + size && (size > loaded_size) + end + # Valid LFS object pointer is a text file consisting of # version # oid @@ -159,10 +159,14 @@ module Gitlab nil end - def truncated? - size && (size > loaded_size) + def external_storage + return unless lfs_pointer? + + :lfs end + alias_method :external_size, :lfs_size + private def has_lfs_version_key? diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 3a73697dc5d..f9a9b767ef4 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -192,6 +192,10 @@ module Gitlab Commit.diff_from_parent(raw_commit, options) end + def deltas + @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + end + def has_zero_stats? stats.total.zero? rescue diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 019be151353..31d1b66b4f7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -183,6 +183,8 @@ module Gitlab when Gitaly::CommitDiffResponse init_from_gitaly(raw_diff) prune_diff_if_eligible(collapse) + when Gitaly::CommitDelta + init_from_gitaly(raw_diff) when nil raise "Nil as raw diff passed" else @@ -278,15 +280,15 @@ module Gitlab end end - def init_from_gitaly(diff_msg) - @diff = diff_msg.raw_chunks.join - @new_path = encode!(diff_msg.to_path.dup) - @old_path = encode!(diff_msg.from_path.dup) - @a_mode = diff_msg.old_mode.to_s(8) - @b_mode = diff_msg.new_mode.to_s(8) - @new_file = diff_msg.from_id == BLANK_SHA - @renamed_file = diff_msg.from_path != diff_msg.to_path - @deleted_file = diff_msg.to_id == BLANK_SHA + def init_from_gitaly(msg) + @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) + @new_path = encode!(msg.to_path.dup) + @old_path = encode!(msg.from_path.dup) + @a_mode = msg.old_mode.to_s(8) + @b_mode = msg.new_mode.to_s(8) + @new_file = msg.from_id == BLANK_SHA + @renamed_file = msg.from_path != msg.to_path + @deleted_file = msg.to_id == BLANK_SHA end def prune_diff_if_eligible(collapse = false) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 4e45ec7c174..bcbad8ec829 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -15,7 +15,6 @@ module Gitlab @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file @all_diffs = !!options.fetch(:all_diffs, false) @no_collapse = !!options.fetch(:no_collapse, true) - @deltas_only = !!options.fetch(:deltas_only, false) @line_count = 0 @byte_count = 0 @@ -27,8 +26,6 @@ module Gitlab if @populated # @iterator.each is slower than just iterating the array in place @array.each(&block) - elsif @deltas_only - each_delta(&block) else Gitlab::GitalyClient.migrate(:commit_raw_diffs) do each_patch(&block) @@ -81,14 +78,6 @@ module Gitlab files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes end - def each_delta - @iterator.each_delta.with_index do |delta, i| - diff = Gitlab::Git::Diff.new(delta) - - yield @array[i] = diff - end - end - def each_patch @iterator.each_with_index do |raw, i| # First yield cached Diff instances from @array diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7dac9f6149..256318cb833 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,17 +45,13 @@ module Gitlab # Default branch in the repository def root_ref - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # @root_ref ||= Gitlab::GitalyClient.migrate(:root_ref) do |is_enabled| - # if is_enabled - # gitaly_ref_client.default_branch_name - # else - @root_ref ||= discover_default_branch - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + @root_ref ||= gitaly_migrate(:root_ref) do |is_enabled| + if is_enabled + gitaly_ref_client.default_branch_name + else + discover_default_branch + end + end end # Alias to old method for compatibility @@ -72,17 +68,13 @@ module Gitlab # Returns an Array of branch names # sorted by name ASC def branch_names - # Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # if is_enabled - # gitaly_ref_client.branch_names - # else - branches.map(&:name) - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + gitaly_migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names + else + branches.map(&:name) + end + end end # Returns an Array of Branches @@ -122,30 +114,43 @@ module Gitlab # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end # Returns an Array of tag names def tag_names - # Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| - # NOTE: This feature is intentionally disabled until - # https://gitlab.com/gitlab-org/gitaly/issues/179 is resolved - # if is_enabled - # gitaly_ref_client.tag_names - # else - rugged.tags.map { |t| t.name } - # end - # end - rescue GRPC::BadStatus => e - raise CommandError.new(e) + gitaly_migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names + else + rugged.tags.map { |t| t.name } + end + end end # Returns an Array of Tags @@ -253,7 +258,7 @@ module Gitlab 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(prefix, storage_path, format), - 'CommitId' => commit.id, + 'CommitId' => commit.id } end @@ -451,7 +456,7 @@ module Gitlab # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) - Gitlab::GitalyClient::Commit.is_ancestor(self, from, to) + gitaly_commit_client.is_ancestor(from, to) end # Return an array of Diff objects that represent the diff @@ -494,7 +499,10 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :date(default) or :topo + # :order is the commits order and allowed value is :none (default), :date, + # :topo, or any combination of them (in an array). Commit ordering types + # are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) actual_options = options.dup @@ -522,11 +530,8 @@ module Gitlab end end - if actual_options[:order] == :topo - walker.sorting(Rugged::SORT_TOPO) - else - walker.sorting(Rugged::SORT_NONE) - end + sort_type = rugged_sort_type(actual_options[:order]) + walker.sorting(sort_type) commits = [] offset = actual_options[:skip] @@ -872,27 +877,6 @@ module Gitlab rugged.remotes[remote_name].push(refspecs) end - # Merge the +source_name+ branch into the +target_name+ branch. This is - # equivalent to `git merge --no_ff +source_name+`, since a merge commit - # is always created. - def merge(source_name, target_name, options = {}) - our_commit = rugged.branches[target_name].target - their_commit = rugged.branches[source_name].target - - raise "Invalid merge target" if our_commit.nil? - raise "Invalid merge source" if their_commit.nil? - - merge_index = rugged.merge_commits(our_commit, their_commit) - return false if merge_index.conflicts? - - actual_options = options.merge( - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - update_ref: "refs/heads/#{target_name}" - ) - Rugged::Commit.create(rugged, actual_options) - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -1128,56 +1112,6 @@ module Gitlab end end - def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) - git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) - - # Put files into a directory before archiving - prefix = "#{archive_name(treeish)}/" - git_archive_cmd << "--prefix=#{prefix}" - - # Format defaults to tar - git_archive_cmd << "--format=#{format}" if format - - git_archive_cmd += %W(-- #{treeish}) - - open(filename, 'w') do |file| - # Create a pipe to act as the '|' in 'git archive ... | gzip' - pipe_rd, pipe_wr = IO.pipe - - # Get the compression process ready to accept data from the read end - # of the pipe - compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) - # The read end belongs to the compression process now; we should - # close our file descriptor for it. - pipe_rd.close - - # Start 'git archive' and tell it to write into the write end of the - # pipe. - git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) - # The write end belongs to 'git archive' now; close it. - pipe_wr.close - - # When 'git archive' and the compression process are finished, we are - # done. - Process.waitpid(git_archive_pid) - raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? - Process.waitpid(compress_pid) - raise "#{compress_cmd.join(' ')} failed" unless $?.success? - end - end - - def nice(cmd) - nice_cmd = %w(nice -n 20) - unless unsupported_platform? - nice_cmd += %w(ionice -c 2 -n 7) - end - nice_cmd + cmd - end - - def unsupported_platform? - %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? - end - # Returns true if the index entry has the special file mode that denotes # a submodule. def submodule?(index_entry) @@ -1273,6 +1207,32 @@ module Gitlab def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + end + + def gitaly_migrate(method, &block) + Gitlab::GitalyClient.migrate(method, &block) + rescue GRPC::NotFound => e + raise NoRepository.new(e) + rescue GRPC::BadStatus => e + raise CommandError.new(e) + end + + # Returns the `Rugged` sorting type constant for one or more given + # sort types. Valid keys are `:none`, `:topo`, and `:date`, or an array + # containing more than one of them. `:date` uses a combination of date and + # topological sorting to closer mimic git's native ordering. + def rugged_sort_type(sort_type) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE | Rugged::SORT_TOPO + } + + @rugged_sort_types.fetch(sort_type, Rugged::SORT_NONE) + end end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b722d8a9f56..d41256d9a84 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -35,7 +35,7 @@ module Gitlab type: entry[:type], mode: entry[:filemode].to_s(8), path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha, + commit_id: sha ) end end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 6babea144c7..742118b76a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -1,43 +1,30 @@ module Gitlab class GitPostReceive include Gitlab::Identifier - attr_reader :repo_path, :identifier, :changes, :project + attr_reader :project, :identifier, :changes - def initialize(repo_path, identifier, changes) - repo_path.gsub!(/\.git\z/, '') - repo_path.gsub!(/\A\//, '') - - @repo_path = repo_path + def initialize(project, identifier, changes) + @project = project @identifier = identifier @changes = deserialize_changes(changes) - - retrieve_project_and_type - end - - def wiki? - @type == :wiki - end - - def regular_project? - @type == :project end def identify(revision) super(identifier, project, revision) end - private + def changes_refs + return enum_for(:changes_refs) unless block_given? - def retrieve_project_and_type - @type = :project - @project = Project.find_by_full_path(@repo_path) + changes.each do |change| + oldrev, newrev, ref = change.strip.split(' ') - if @repo_path.end_with?('.wiki') && !@project - @type = :wiki - @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) + yield oldrev, newrev, ref end end + private + def deserialize_changes(changes) changes = utf8_encode_changes(changes) changes.lines diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index b7f39f3ef0b..15c57420fb2 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,34 +5,55 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - class << self - def diff_from_parent(commit, options = {}) - repository = commit.project.repository - gitaly_repo = repository.gitaly_repository - stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: repository.gitaly_channel) - parent = commit.parents[0] - parent_id = parent ? parent.id : EMPTY_TREE_ID - request = Gitaly::CommitDiffRequest.new( - repository: gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id - ) - - Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) - end + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @repository = repository + end + + def is_ancestor(ancestor_id, child_id) + stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: @repository.gitaly_channel) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + stub.commit_is_ancestor(request).value + end + + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) + Gitlab::Git::DiffCollection.new(response, options) + end - def is_ancestor(repository, ancestor_id, child_id) - gitaly_repo = repository.gitaly_repository - stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) - request = Gitaly::CommitIsAncestorRequest.new( - repository: gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) + def commit_deltas(commit) + request_params = commit_diff_request_params(commit) - stub.commit_is_ancestor(request).value + response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def diff_service_stub + Gitaly::Diff::Stub.new(nil, nil, channel_override: @repository.gitaly_channel) + end end end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index d3c0743db4e..f6c77ef1a3e 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -11,7 +11,9 @@ module Gitlab def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - stub.find_default_branch_name(request).name.gsub(/^refs\/heads\//, '') + branch_name = stub.find_default_branch_name(request).name + + Gitlab::Git.branch_name(branch_name) end def branch_names @@ -34,6 +36,14 @@ module Gitlab stub.find_ref_name(request).name end + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + private def consume_refs_response(response, prefix:) diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 4acd297f5cb..86d055d3533 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -6,7 +6,7 @@ module Gitlab Gitaly::Repository.new( path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), storage_name: repository_storage, - relative_path: relative_path, + relative_path: relative_path ) end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 6f5ac4dac0d..977cd0423ba 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -10,7 +10,7 @@ module Gitlab description: description, state: state, author_id: author_id, - assignee_id: assignee_id, + assignee_ids: Array(assignee_id), created_at: raw_data.created_at, updated_at: raw_data.updated_at } diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb new file mode 100644 index 00000000000..07c0abcce23 --- /dev/null +++ b/lib/gitlab/gl_repository.rb @@ -0,0 +1,20 @@ +module Gitlab + module GlRepository + def self.gl_repository(project, is_wiki) + "#{is_wiki ? 'wiki' : 'project'}-#{project.id}" + end + + def self.parse(gl_repository) + match_data = /\A(project|wiki)-([1-9][0-9]*)\z/.match(gl_repository) + unless match_data + raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" + end + + type, id = match_data.captures + project = Project.find_by(id: id) + wiki = type == 'wiki' + + [project, wiki] + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5ab84266b7d..26473f99bc3 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -10,6 +10,8 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') + gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.gitlab_url = Gitlab.config.gitlab.url if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..1b43440673c 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -92,13 +108,13 @@ module Gitlab end issue = Issue.create!( - iid: raw_issue['id'], - project_id: project.id, - title: raw_issue['title'], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_ids: [assignee_id], + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d787d5db4a0..83bc230df3e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -13,6 +13,8 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end + attr_reader :blob_name + def initialize(blob_name, blob_content, repository: nil) @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @@ -21,16 +23,9 @@ module Gitlab end def highlight(text, continue: true, plain: false) - if plain - hl_lexer = Rouge::Lexers::PlainText - continue = false - else - hl_lexer = self.lexer - end - - @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe - rescue - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + highlighted_text = highlight_text(text, continue: continue, plain: plain) + highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text end def lexer @@ -50,5 +45,27 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end + + def highlight_text(text, continue: true, plain: false) + if plain + highlight_plain(text) + else + highlight_rich(text, continue: continue) + end + end + + def highlight_plain(text) + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + def highlight_rich(text, continue: true) + @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + rescue + highlight_plain(text) + end + + def link_dependencies(text, highlighted_text) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb new file mode 100644 index 00000000000..3411516319f --- /dev/null +++ b/lib/gitlab/i18n.rb @@ -0,0 +1,26 @@ +module Gitlab + module I18n + extend self + + AVAILABLE_LANGUAGES = { + 'en' => 'English', + 'es' => 'Español', + 'de' => 'Deutsch' + }.freeze + + def available_locales + AVAILABLE_LANGUAGES.keys + end + + def set_locale(current_user) + requested_locale = current_user&.preferred_language || ::I18n.default_locale + locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = locale + end + + def reset_locale + FastGettext.set_locale(::I18n.default_locale) + ::I18n.locale = ::I18n.default_locale + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 8b327cfc226..27d5a9198b6 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.6'.freeze + VERSION = '0.1.7'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 899a6567768..d0f3cf2b514 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,9 +39,8 @@ project_tree: - :author - :events - :statuses - - triggers: - - :trigger_schedule - - :deploy_keys + - :triggers + - :pipeline_schedules - :services - :hooks - protected_branches: @@ -53,10 +52,6 @@ project_tree: # Only include the following attributes for the models specified. included_attributes: - project: - - :description - - :visibility_level - - :archived user: - :id - :email @@ -66,6 +61,32 @@ included_attributes: # Do not include the following attributes for the models specified. excluded_attributes: + project: + - :name + - :path + - :namespace_id + - :creator_id + - :import_url + - :import_status + - :avatar + - :import_type + - :import_source + - :import_error + - :mirror + - :runners_token + - :repository_storage + - :repository_read_only + - :lfs_enabled + - :import_jid + - :created_at + - :updated_at + - :import_jid + - :import_jid + - :id + - :star_count + - :last_activity_at + - :last_repository_updated_at + - :last_repository_check_at snippets: - :expired_at merge_request_diff: @@ -87,10 +108,11 @@ methods: - :type statuses: - :type - - :gl_project_id services: - :type merge_request_diff: - :utf8_st_diffs merge_requests: - :diff_head_sha + project: + - :description_html diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 2e349b5f9a9..84ab1977dfa 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -71,14 +71,14 @@ module Gitlab def restore_project return @project unless @tree_hash - @project.update(project_params) + @project.update_columns(project_params) @project end def project_params @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations - value.is_a?(Array) || key == key.singularize + value.respond_to?(:each) && !Project.column_names.include?(key) end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index a1e7159fe42..eb7f5120592 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -15,7 +15,10 @@ module Gitlab # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + attributes = @attributes_finder.find(:project) + project_attributes = attributes.is_a?(Hash) ? attributes[:project] : {} + + project_attributes.merge(include: build_hash(@tree)) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 4a54e7ef2e7..19e23a4715f 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -5,7 +5,7 @@ module Gitlab pipelines: 'Ci::Pipeline', statuses: 'commit_status', triggers: 'Ci::Trigger', - trigger_schedule: 'Ci::TriggerSchedule', + pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', @@ -15,7 +15,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 3a7af363548..4a6091488c8 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -38,7 +38,7 @@ module Gitlab url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], headers: Hash.new { |h, k| h[k] = [] }, - created_at: created_at, + created_at: created_at } end end @@ -64,7 +64,7 @@ module Gitlab tty: true, stdin: true, stdout: true, - stderr: true, + stderr: true }.to_query + '&' + EXEC_COMMAND case url.scheme diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 46deea3cc9f..6fdf68641e2 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -39,7 +39,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption, + encryption: encryption ) opts.merge!(auth_options) if has_auth? diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 6e42d8941fb..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::BuildService.new(nil, user_params).execute + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index e67acf28c94..31a24460f0f 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -4,21 +4,13 @@ module Gitlab # Public: Converts the provided markup into HTML. # # input - the source text in a markup format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref # def self.render(file_name, input, context) html = GitHub::Markup.render(file_name, input). force_encoding(input.encoding) + context[:pipeline] = :markup - html = Banzai.post_process(html, context) - - filter = Banzai::Filter::SanitizationFilter.new(html) - html = filter.call.to_s + html = Banzai.render(html, context) html.html_safe end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 0b8959f2fb9..561aa9e162c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -82,26 +82,14 @@ module Gitlab private def blobs - @blobs ||= begin - blobs = project.repository.search_files_by_content(query, repository_ref).first(100) - found_file_names = Set.new - - results = blobs.map do |blob| - blob = self.class.parse_search_result(blob) - found_file_names << blob.filename - - [blob.filename, blob] - end - - project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| - results << [filename, nil] unless found_file_names.include?(filename) - end + return [] unless Ability.allowed?(@current_user, :download_code, @project) - results.sort_by(&:first) - end + @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) end def wiki_blobs + return [] unless Ability.allowed?(@current_user, :read_wiki, @project) + @wiki_blobs ||= begin if project.wiki_enabled? && query.present? project_wiki = ProjectWiki.new(project) diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb new file mode 100644 index 00000000000..2a2eb4ae57f --- /dev/null +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -0,0 +1,26 @@ +module Gitlab + module Prometheus + module Queries + class BaseQuery + attr_accessor :client + delegate :query_range, :query, to: :client, prefix: true + + def raw_memory_usage_query(environment_slug) + %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} + end + + def raw_cpu_usage_query(environment_slug) + %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} + end + + def initialize(client) + @client = client + end + + def query(*args) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb new file mode 100644 index 00000000000..2cc08731f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -0,0 +1,26 @@ +module Gitlab::Prometheus::Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug + + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), + + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb new file mode 100644 index 00000000000..01d756d7284 --- /dev/null +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -0,0 +1,20 @@ +module Gitlab::Prometheus::Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus_client.rb index 62239779454..5b51a1779dd 100644 --- a/lib/gitlab/prometheus.rb +++ b/lib/gitlab/prometheus_client.rb @@ -2,7 +2,7 @@ module Gitlab PrometheusError = Class.new(StandardError) # Helper methods to interact with Prometheus network services & resources - class Prometheus + class PrometheusClient attr_reader :api_url def initialize(api_url:) @@ -13,18 +13,18 @@ module Gitlab json_api_get('query', query: '1') end - def query(query) + def query(query, time: Time.now) get_result('vector') do - json_api_get('query', query: query) + json_api_get('query', query: query, time: time.to_f) end end - def query_range(query, start: 8.hours.ago) + def query_range(query, start: 8.hours.ago, stop: Time.now) get_result('matrix') do json_api_get('query_range', query: query, start: start.to_f, - end: Time.now.utc.to_f, + end: stop.to_f, step: 1.minute.to_i) end end @@ -50,6 +50,12 @@ module Gitlab def get(url) handle_response(HTTParty.get(url)) + rescue SocketError + raise PrometheusError, "Can't connect to #{url}" + rescue OpenSSL::SSL::SSLError + raise PrometheusError, "#{url} contains invalid SSL data" + rescue HTTParty::Error + raise PrometheusError, "Network connection error" end def handle_response(response) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 08b061d5e31..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 4b1d828c45c..878e03f61d7 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -2,18 +2,29 @@ module Gitlab module RepoPath NotFoundError = Class.new(StandardError) - def self.strip_storage_path(repo_path) - result = nil + def self.parse(repo_path) + project_path = strip_storage_path(repo_path.sub(/\.git\z/, ''), fail_on_not_found: false) + project = Project.find_by_full_path(project_path) + if project_path.end_with?('.wiki') && !project + project = Project.find_by_full_path(project_path.chomp('.wiki')) + wiki = true + else + wiki = false + end + + [project, wiki] + end + + def self.strip_storage_path(repo_path, fail_on_not_found: true) + result = repo_path - Gitlab.config.repositories.storages.values.each do |params| - storage_path = params['path'] - if repo_path.start_with?(storage_path) - result = repo_path.sub(storage_path, '') - break - end + storage = Gitlab.config.repositories.storages.values.find do |params| + repo_path.start_with?(params['path']) end - if result.nil? + if storage + result = result.sub(storage['path'], '') + elsif fail_on_not_found raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 117fc508135..2442c2ded3b 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -11,7 +11,7 @@ module Gitlab Raven.user_context( id: current_user.id, email: current_user.email, - username: current_user.username, + username: current_user.username ) end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 36a871e5bbc..b1d6ea665b7 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -83,7 +83,7 @@ module Gitlab # Timeout should be less than 900 ideally, to prevent the memory killer # to silently kill the process without knowing we are timing out here. output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, '800']) + storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]) raise Error, output unless status.zero? true end @@ -99,7 +99,7 @@ module Gitlab # fetch_remote("gitlab/gitlab-ci", "upstream") # def fetch_remote(storage, name, remote, forced: false, no_tags: false) - args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, '800'] + args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, "#{Gitlab.config.gitlab_shell.git_timeout}"] args << '--force' if forced args << '--no-tags' if no_tags diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb index 60d35be2599..12a385f90fd 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -1,16 +1,19 @@ module Gitlab module SlashCommands class CommandDefinition - attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + attr_accessor :name, :aliases, :description, :explanation, :params, + :condition_block, :parse_params_block, :action_block def initialize(name, attributes = {}) @name = name - @aliases = attributes[:aliases] || [] - @description = attributes[:description] || '' - @params = attributes[:params] || [] + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @explanation = attributes[:explanation] || '' + @params = attributes[:params] || [] @condition_block = attributes[:condition_block] - @action_block = attributes[:action_block] + @parse_params_block = attributes[:parse_params_block] + @action_block = attributes[:action_block] end def all_names @@ -28,14 +31,20 @@ module Gitlab context.instance_exec(&condition_block) end + def explain(context, opts, arg) + return unless available?(opts) + + if explanation.respond_to?(:call) + execute_block(explanation, context, arg) + else + explanation + end + end + def execute(context, opts, arg) return if noop? || !available?(opts) - if arg.present? - context.instance_exec(arg, &action_block) - elsif action_block.arity == 0 - context.instance_exec(&action_block) - end + execute_block(action_block, context, arg) end def to_h(opts) @@ -52,6 +61,23 @@ module Gitlab params: params } end + + private + + def execute_block(block, context, arg) + if arg.present? + parsed = parse_params(arg, context) + context.instance_exec(parsed, &block) + elsif block.arity == 0 + context.instance_exec(&block) + end + end + + def parse_params(arg, context) + return arg unless parse_params_block + + context.instance_exec(arg, &parse_params_block) + end end end end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb index 50b0937d267..614bafbe1b2 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/slash_commands/dsl.rb @@ -44,6 +44,22 @@ module Gitlab @params = params end + # Allows to give an explanation of what the command will do when + # executed. This explanation is shown when rendering the Markdown + # preview. + # + # Example: + # + # explanation do |arguments| + # "Adds label(s) #{arguments.join(' ')}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def explanation(text = '', &block) + @explanation = block_given? ? block : text + end + # Allows to define conditions that must be met in order for the command # to be returned by `.command_names` & `.command_definitions`. # It accepts a block that will be evaluated with the context given to @@ -61,6 +77,24 @@ module Gitlab @condition_block = block end + # Allows to perform initial parsing of parameters. The result is passed + # both to `command` and `explanation` blocks, instead of the raw + # parameters. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # parse_params do |raw| + # raw.strip + # end + # command :command_key do |parsed| + # # Awesome code block + # end + def parse_params(&block) + @parse_params_block = block + end + # Registers a new command which is recognizeable from body of email or # comment. # It accepts aliases and takes a block. @@ -75,11 +109,13 @@ module Gitlab definition = CommandDefinition.new( name, - aliases: aliases, - description: @description, - params: @params, - condition_block: @condition_block, - action_block: block + aliases: aliases, + description: @description, + explanation: @explanation, + params: @params, + condition_block: @condition_block, + parse_params_block: @parse_params_block, + action_block: block ) self.command_definitions << definition @@ -89,8 +125,14 @@ module Gitlab end @description = nil + @explanation = nil @params = nil @condition_block = nil + @parse_params_block = nil + end + + def definition_by_name(name) + command_definitions_by_name[name.to_sym] end end end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb new file mode 100644 index 00000000000..94fba0a221a --- /dev/null +++ b/lib/gitlab/string_range_marker.rb @@ -0,0 +1,102 @@ +module Gitlab + class StringRangeMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(marker_ranges) + return rich_line unless marker_ranges + + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + + offset = 0 + # Mark each range + rich_marker_ranges.each_with_index do |range, i| + offset_range = (range.begin + offset)..(range.end + offset) + original_text = rich_line[offset_range] + + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + + rich_line[offset_range] = text + + offset += text.length - original_text.length + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + end +end diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb new file mode 100644 index 00000000000..7ebf1c0428c --- /dev/null +++ b/lib/gitlab/string_regex_marker.rb @@ -0,0 +1,13 @@ +module Gitlab + class StringRegexMarker < StringRangeMarker + def mark(regex, group: 0, &block) + regex_match = raw_line.match(regex) + return rich_line unless regex_match + + begin_index, end_index = regex_match.offset(group) + name_range = begin_index..(end_index - 1) + + super([name_range], &block) + end + end +end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6aca6db3123..4382cf7b12f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -23,6 +23,7 @@ module Gitlab ci_pipelines: ::Ci::Pipeline.count, ci_runners: ::Ci::Runner.count, ci_triggers: ::Ci::Trigger.count, + ci_pipeline_schedules: ::Ci::PipelineSchedule.count, deploy_keys: DeployKey.count, deployments: Deployment.count, environments: Environment.count, @@ -51,6 +52,7 @@ module Gitlab def license_usage_data usage_data = { uuid: current_application_settings.uuid, + hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, recorded_at: Time.now, diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 54728e5ff0e..3b922da7ced 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -38,15 +38,23 @@ module Gitlab end end + def can_delete_branch?(ref) + return false unless can_access_git? + + if ProtectedBranch.protected?(project, ref) + user.can?(:delete_protected_branch, project) + else + user.can?(:push_code, project) + end + end + def can_push_to_branch?(ref) return false unless can_access_git? if ProtectedBranch.protected?(project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - has_access = project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) - - has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) else user.can?(:push_code, project) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e6e40f6945d..351e2b10595 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -16,15 +16,17 @@ module Gitlab SECRET_LENGTH = 32 class << self - def git_http_ok(repository, user, action) + def git_http_ok(repository, is_wiki, user, action) + project = repository.project repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), - RepoPath: repo_path, + GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), + RepoPath: repo_path } if Gitlab.config.gitaly.enabled - address = Gitlab::GitalyClient.get_address(repository.project.repository_storage) + address = Gitlab::GitalyClient.get_address(project.repository_storage) params[:Repository] = repository.gitaly_repository.to_h feature_enabled = case action.to_s @@ -49,7 +51,7 @@ module Gitlab { StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", LfsOid: oid, - LfsSize: size, + LfsSize: size } end @@ -60,7 +62,7 @@ module Gitlab def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id, + 'BlobId' => blob.id } [ @@ -125,7 +127,7 @@ module Gitlab 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], 'Header' => terminal[:headers], - 'MaxSessionTime' => terminal[:max_session_time], + 'MaxSessionTime' => terminal[:max_session_time] } } details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) @@ -163,12 +165,12 @@ module Gitlab encoded_message, secret, true, - { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' } ) end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 2301ec9b228..99b3168d9eb 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 5293f5af12d..87ca39b079b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -19,8 +19,9 @@ namespace :gemojione do entry = { category: emoji_hash['category'], moji: emoji_hash['moji'], + description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), - digest: hash_digest, + digest: hash_digest } resultant_emoji_map[name] = entry diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake new file mode 100644 index 00000000000..0aa21a4bd13 --- /dev/null +++ b/lib/tasks/gettext.rake @@ -0,0 +1,14 @@ +require "gettext_i18n_rails/tasks" + +namespace :gettext do + # Customize list of translatable files + # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files + def files_to_translate + folders = %W(app lib config #{locale_path}).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue coffee handlebars hbs mustache).join(',') + + Dir.glob( + "{#{folders}}/**/*.{#{exts}}" + ) + end +end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 5476438b8fa..139ab70e125 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -65,6 +65,7 @@ namespace :gitlab do migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines .map { |file| Rails.root.join(file.strip).to_s } .select { |file| File.file?(file) } + .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 8079c6e416c..3c5bc0146a1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,21 +1,24 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| + require 'toml' + warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version - repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do + create_gitaly_configuration run_command!([command]) end end @@ -33,5 +36,39 @@ namespace :gitlab do puts TOML.dump(storage: config) end + + private + + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only write a config.toml file in the most common and simplest case: the + # case where we have exactly one Gitaly process and we are sure it is + # running locally because it uses a Unix socket. + def create_gitaly_configuration + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + File.open("config.toml", "w") do |f| + f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + end + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index a2a2db487b7..e3883278886 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -16,6 +16,8 @@ namespace :gitlab do redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a # check Git version git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a + # check Go version + go_version = run_and_match(%w(go version), /go version (.+)/).to_a puts "" puts "System information".color(:yellow) @@ -30,6 +32,7 @@ namespace :gitlab do puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" + puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 95687066819..ee2cdcdea1b 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -41,8 +41,14 @@ namespace :gitlab do # Generate config.yml based on existing gitlab settings File.open("config.yml", "w+") {|f| f.puts config.to_yaml} - # Launch installation process - system(*%w(bin/install) + repository_storage_paths_args) + [ + %w(bin/install) + repository_storage_paths_args, + %w(bin/compile) + ].each do |cmd| + unless Kernel.system(*cmd) + raise "command failed: #{cmd.join(' ')}" + end + end end # (Re)create hooks diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index cb2adc81c9d..59c32bbe7a4 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,7 +5,7 @@ namespace :gitlab do end def update(template) - sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) unless clone_repository(template.repo_url, dir) @@ -45,6 +45,10 @@ namespace :gitlab do Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ + ), + Template.new( + "https://gitlab.com/gitlab-org/Dockerfile.git", + /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ ) ].freeze diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index a00b02188cf..e7ac0b5859f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,16 +1,16 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version - repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index a9dad6a1bf0..bc76d7edc55 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,67 +1,5 @@ require 'benchmark' require 'rainbow/ext/string' -require_relative '../gitlab/shell_adapter' -require_relative '../gitlab/github_import/importer' - -class NewImporter < ::Gitlab::GithubImport::Importer - def execute - # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. - puts 'Importing repository...'.color(:aqua) - import_repository unless project.repository_exists? - - puts 'Importing labels...'.color(:aqua) - import_labels - - puts 'Importing milestones...'.color(:aqua) - import_milestones - - puts 'Importing pull requests...'.color(:aqua) - import_pull_requests - - puts 'Importing issues...'.color(:aqua) - import_issues - - puts 'Importing issue comments...'.color(:aqua) - import_comments(:issues) - - puts 'Importing pull request comments...'.color(:aqua) - import_comments(:pull_requests) - - puts 'Importing wiki...'.color(:aqua) - import_wiki - - # Gitea doesn't have a Release API yet - # See https://github.com/go-gitea/gitea/issues/330 - unless project.gitea_import? - import_releases - end - - handle_errors - - project.repository.after_import - project.import_finish - - true - end - - def import_repository - begin - raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) - - project.create_repository - project.repository.add_remote(project.import_type, project.import_url) - project.repository.set_remote_as_mirror(project.import_type) - project.repository.fetch_remote(project.import_type, forced: true) - rescue => e - # Expire cache to prevent scenarios such as: - # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true - # 2. Retried import, repo is broken or not imported but +exists?+ still returns true - project.repository.expire_content_cache if project.repository_exists? - - raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" - end - end -end class GithubImport def self.run!(*args) @@ -69,14 +7,14 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @token = token + @options = { url: 'https://api.github.com', token: token, verbose: true } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first end def run! - @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! raise 'No repo found!' unless @repo @@ -90,25 +28,24 @@ class GithubImport private def show_warning! - puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch - puts 'Starting the import...'.color(:green) + puts 'Starting the import (this could take a while)'.color(:green) end def import! - import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") - @project.update(import_url: import_url) - @project.import_start timings = Benchmark.measure do - NewImporter.new(@project).execute + Github::Import.new(@project, @options).execute end puts "Import finished. Timings: #{timings}".color(:green) + + @project.import_finish end def new_project @@ -116,17 +53,17 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Project.create!( - import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + Projects::CreateService.new( + @current_user, name: name, path: name, - description: @repo.description, - namespace: namespace, + description: @repo['description'], + namespace_id: namespace.id, visibility_level: visibility_level, import_type: 'github', - import_source: @repo.full_name, - creator: @current_user - ) + import_source: @repo['full_name'], + skip_wiki: @repo['has_wiki'] + ).execute end end @@ -134,7 +71,6 @@ class GithubImport return @current_user.namespace if names == @current_user.namespace_path return @current_user.namespace unless @current_user.can_create_group? - names = params[:target_namespace].presence || names full_path_namespace = Namespace.find_by_full_path(names) return full_path_namespace if full_path_namespace @@ -159,13 +95,13 @@ class GithubImport end def visibility_level - @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end end class GithubRepos - def initialize(token, current_user, github_repo) - @token = token + def initialize(options, current_user, github_repo) + @options = options @current_user = current_user @github_repo = github_repo end @@ -174,17 +110,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo[:id].to_s.bright} ".color(:green) - puts "- Name: #{repo[:full_name]}".color(:green) + print "ID: #{repo['id'].to_s.bright}".color(:green) + print "\tName: #{repo['full_name']}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo[:id] == repo_id } + repos.find { |repo| repo['id'] == repo_id } end def found_github_repo - repos.find { |repo| repo[:full_name] == @github_repo } + repos.find { |repo| repo['full_name'] == @github_repo } end def repo_id @@ -192,11 +128,7 @@ class GithubRepos end def repos - @repos ||= client.repos - end - - def client - @client ||= Gitlab::GithubImport::Client.new(@token, {}) + Github::Repositories.new(@options).fetch end end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 6ded519aff2..761f275d42a 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -1,7 +1,9 @@ require Rails.root.join('db/migrate/limits_to_mysql') +require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do puts "Adding limits to schema.rb for mysql" LimitsToMysql.new.up + MarkdownCacheLimitsToMysql.new.up end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 8938bc515f5..4108cee08b4 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -4,6 +4,7 @@ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lowe require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') +require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do @@ -11,4 +12,5 @@ task setup_postgresql: :environment do AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up IndexRoutesPathForLike.new.up + IndexRedirectRoutesPathForLike.new.up end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 602c60be828..2eddcb3c777 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -60,7 +60,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %w(rake gitlab:setup), - %w(rspec spec), + %w(rspec spec) ] run_commands(cmds) end |