Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorDouwe Maan <douwe@gitlab.com>2018-02-06 19:08:06 +0300
committerDouwe Maan <douwe@gitlab.com>2018-02-06 19:08:06 +0300
commit5db5a9cbd1192d776874a92e0a253c605f3c4417 (patch)
tree67be95609ce038ec9c2fbdc45fb491bb223c7137 /lib
parenta29f0c28fd07ba14f0d0e5fb9c878a2eb117e388 (diff)
parentba62493009c2360018709b660956a6173f3e1515 (diff)
Merge branch 'master' into 'zj-wiki-page-versions'
# Conflicts: # Gemfile # Gemfile.lock
Diffstat (limited to 'lib')
-rw-r--r--lib/api/access_requests.rb6
-rw-r--r--lib/api/api.rb1
-rw-r--r--lib/api/api_guard.rb6
-rw-r--r--lib/api/applications.rb27
-rw-r--r--lib/api/branches.rb2
-rw-r--r--lib/api/entities.rb41
-rw-r--r--lib/api/helpers/internal_helpers.rb15
-rw-r--r--lib/api/helpers/runner.rb6
-rw-r--r--lib/api/issues.rb6
-rw-r--r--lib/api/members.rb13
-rw-r--r--lib/api/merge_requests.rb6
-rw-r--r--lib/api/pipelines.rb2
-rw-r--r--lib/api/projects.rb2
-rw-r--r--lib/api/runner.rb6
-rw-r--r--lib/api/templates.rb8
-rw-r--r--lib/api/triggers.rb2
-rw-r--r--lib/api/users.rb2
-rw-r--r--lib/api/v3/branches.rb2
-rw-r--r--lib/api/v3/issues.rb6
-rw-r--r--lib/api/v3/members.rb15
-rw-r--r--lib/api/v3/merge_requests.rb4
-rw-r--r--lib/api/v3/pipelines.rb2
-rw-r--r--lib/api/v3/projects.rb4
-rw-r--r--lib/api/v3/templates.rb8
-rw-r--r--lib/api/v3/triggers.rb2
-rw-r--r--lib/backup/artifacts.rb2
-rw-r--r--lib/banzai/color_parser.rb44
-rw-r--r--lib/banzai/filter/color_filter.rb31
-rw-r--r--lib/banzai/filter/emoji_filter.rb4
-rw-r--r--lib/banzai/filter/gollum_tags_filter.rb4
-rw-r--r--lib/banzai/pipeline/broadcast_message_pipeline.rb1
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb1
-rw-r--r--lib/container_registry/registry.rb2
-rw-r--r--lib/email_template_interceptor.rb4
-rw-r--r--lib/extracts_path.rb4
-rw-r--r--lib/gitaly/server.rb43
-rw-r--r--lib/gitlab/asciidoc.rb8
-rw-r--r--lib/gitlab/auth.rb6
-rw-r--r--lib/gitlab/background_migration/populate_untracked_uploads.rb4
-rw-r--r--lib/gitlab/background_migration/prepare_untracked_uploads.rb9
-rw-r--r--lib/gitlab/badge/coverage/report.rb2
-rw-r--r--lib/gitlab/badge/coverage/template.rb4
-rw-r--r--lib/gitlab/bare_repository_import/repository.rb2
-rw-r--r--lib/gitlab/checks/change_access.rb38
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata/entry.rb2
-rw-r--r--lib/gitlab/ci/trace.rb12
-rw-r--r--lib/gitlab/current_settings.rb108
-rw-r--r--lib/gitlab/database/grant.rb50
-rw-r--r--lib/gitlab/dependency_linker/composer_json_linker.rb2
-rw-r--r--lib/gitlab/dependency_linker/gemfile_linker.rb2
-rw-r--r--lib/gitlab/dependency_linker/podspec_linker.rb2
-rw-r--r--lib/gitlab/ee_compat_check.rb2
-rw-r--r--lib/gitlab/email/reply_parser.rb2
-rw-r--r--lib/gitlab/file_detector.rb22
-rw-r--r--lib/gitlab/gfm/uploads_rewriter.rb4
-rw-r--r--lib/gitlab/git.rb3
-rw-r--r--lib/gitlab/git/attributes_at_ref_parser.rb14
-rw-r--r--lib/gitlab/git/attributes_parser.rb (renamed from lib/gitlab/git/attributes.rb)53
-rw-r--r--lib/gitlab/git/blame.rb4
-rw-r--r--lib/gitlab/git/blob.rb28
-rw-r--r--lib/gitlab/git/branch.rb2
-rw-r--r--lib/gitlab/git/diff.rb2
-rw-r--r--lib/gitlab/git/info_attributes.rb49
-rw-r--r--lib/gitlab/git/operation_service.rb5
-rw-r--r--lib/gitlab/git/path_helper.rb2
-rw-r--r--lib/gitlab/git/popen.rb2
-rw-r--r--lib/gitlab/git/ref.rb2
-rw-r--r--lib/gitlab/git/remote_mirror.rb24
-rw-r--r--lib/gitlab/git/repository.rb564
-rw-r--r--lib/gitlab/git/repository_mirroring.rb2
-rw-r--r--lib/gitlab/git/tag.rb2
-rw-r--r--lib/gitlab/git/tree.rb2
-rw-r--r--lib/gitlab/git/wiki.rb38
-rw-r--r--lib/gitlab/git/wiki_page.rb3
-rw-r--r--lib/gitlab/gitaly_client.rb54
-rw-r--r--lib/gitlab/gitaly_client/blob_service.rb22
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb26
-rw-r--r--lib/gitlab/gitaly_client/conflicts_service.rb5
-rw-r--r--lib/gitlab/gitaly_client/health_check_service.rb19
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb106
-rw-r--r--lib/gitlab/gitaly_client/ref_service.rb42
-rw-r--r--lib/gitlab/gitaly_client/remote_service.rb27
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb71
-rw-r--r--lib/gitlab/gitaly_client/server_service.rb16
-rw-r--r--lib/gitlab/gitaly_client/wiki_page.rb5
-rw-r--r--lib/gitlab/gitaly_client/wiki_service.rb12
-rw-r--r--lib/gitlab/github_import/importer/pull_requests_importer.rb5
-rw-r--r--lib/gitlab/github_import/representation/diff_note.rb2
-rw-r--r--lib/gitlab/github_import/representation/note.rb2
-rw-r--r--lib/gitlab/gon_helper.rb5
-rw-r--r--lib/gitlab/health_checks/gitaly_check.rb53
-rw-r--r--lib/gitlab/import_export/command_line_util.rb9
-rw-r--r--lib/gitlab/import_export/file_importer.rb2
-rw-r--r--lib/gitlab/import_export/relation_factory.rb5
-rw-r--r--lib/gitlab/import_export/repo_restorer.rb2
-rw-r--r--lib/gitlab/import_export/repo_saver.rb2
-rw-r--r--lib/gitlab/import_export/shared.rb7
-rw-r--r--lib/gitlab/import_export/uploads_saver.rb8
-rw-r--r--lib/gitlab/import_export/wiki_repo_saver.rb2
-rw-r--r--lib/gitlab/kubernetes/helm/pod.rb8
-rw-r--r--lib/gitlab/ldap/auth_hash.rb6
-rw-r--r--lib/gitlab/ldap/config.rb4
-rw-r--r--lib/gitlab/ldap/person.rb4
-rw-r--r--lib/gitlab/legacy_github_import/project_creator.rb4
-rw-r--r--lib/gitlab/metrics.rb4
-rw-r--r--lib/gitlab/metrics/influx_db.rb290
-rw-r--r--lib/gitlab/metrics/method_call.rb38
-rw-r--r--lib/gitlab/metrics/methods.rb129
-rw-r--r--lib/gitlab/metrics/methods/metric_options.rb61
-rw-r--r--lib/gitlab/metrics/null_metric.rb2
-rw-r--r--lib/gitlab/metrics/prometheus.rb93
-rw-r--r--lib/gitlab/metrics/subscribers/action_view.rb24
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb18
-rw-r--r--lib/gitlab/metrics/transaction.rb83
-rw-r--r--lib/gitlab/middleware/go.rb7
-rw-r--r--lib/gitlab/middleware/static.rb2
-rw-r--r--lib/gitlab/o_auth/user.rb2
-rw-r--r--lib/gitlab/performance_bar.rb4
-rw-r--r--lib/gitlab/polling_interval.rb6
-rw-r--r--lib/gitlab/popen.rb27
-rw-r--r--lib/gitlab/popen/runner.rb46
-rw-r--r--lib/gitlab/profiler.rb142
-rw-r--r--lib/gitlab/project_search_results.rb2
-rw-r--r--lib/gitlab/protocol_access.rb6
-rw-r--r--lib/gitlab/query_limiting.rb36
-rw-r--r--lib/gitlab/query_limiting/active_support_subscriber.rb11
-rw-r--r--lib/gitlab/query_limiting/middleware.rb55
-rw-r--r--lib/gitlab/query_limiting/transaction.rb83
-rw-r--r--lib/gitlab/quick_actions/spend_time_and_date_separator.rb2
-rw-r--r--lib/gitlab/recaptcha.rb10
-rw-r--r--lib/gitlab/redis/cache.rb2
-rw-r--r--lib/gitlab/repo_path.rb2
-rw-r--r--lib/gitlab/search_results.rb66
-rw-r--r--lib/gitlab/seeder.rb10
-rw-r--r--lib/gitlab/sentry.rb4
-rw-r--r--lib/gitlab/setup_helper.rb2
-rw-r--r--lib/gitlab/sherlock/file_sample.rb2
-rw-r--r--lib/gitlab/sherlock/middleware.rb2
-rw-r--r--lib/gitlab/sherlock/query.rb4
-rw-r--r--lib/gitlab/snippet_search_results.rb2
-rw-r--r--lib/gitlab/ssh_public_key.rb28
-rw-r--r--lib/gitlab/task_helpers.rb (renamed from lib/tasks/gitlab/task_helpers.rb)9
-rw-r--r--lib/gitlab/upgrader.rb5
-rw-r--r--lib/gitlab/uploads_transfer.rb2
-rw-r--r--lib/gitlab/usage_data.rb8
-rw-r--r--lib/gitlab/visibility_level.rb7
-rw-r--r--lib/gitlab/workhorse.rb26
-rw-r--r--lib/support/nginx/gitlab4
-rw-r--r--lib/support/nginx/gitlab-ssl4
-rw-r--r--lib/system_check/app/git_version_check.rb2
-rw-r--r--lib/system_check/helpers.rb2
-rw-r--r--lib/tasks/flay.rake4
-rw-r--r--lib/tasks/gitlab/backup.rake36
-rw-r--r--lib/tasks/gitlab/check.rake26
-rw-r--r--lib/tasks/gitlab/cleanup.rake10
-rw-r--r--lib/tasks/gitlab/git.rake8
-rw-r--r--lib/tasks/gitlab/gitaly.rake8
-rw-r--r--lib/tasks/gitlab/helpers.rake4
-rw-r--r--lib/tasks/gitlab/info.rake2
-rw-r--r--lib/tasks/gitlab/setup.rake2
-rw-r--r--lib/tasks/gitlab/shell.rake8
-rw-r--r--lib/tasks/gitlab/workhorse.rake2
-rw-r--r--lib/tasks/haml-lint.rake9
-rw-r--r--lib/tasks/migrate/setup_postgresql.rake20
164 files changed, 2527 insertions, 939 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index 374b611f55e..60ae5e6b9a2 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -24,7 +24,7 @@ module API
access_requesters = AccessRequestsFinder.new(source).execute!(current_user)
access_requesters = paginate(access_requesters.includes(:user))
- present access_requesters.map(&:user), with: Entities::AccessRequester, source: source
+ present access_requesters, with: Entities::AccessRequester
end
desc "Requests access for the authenticated user to a #{source_type}." do
@@ -36,7 +36,7 @@ module API
access_requester = source.request_access(current_user)
if access_requester.persisted?
- present access_requester.user, with: Entities::AccessRequester, access_requester: access_requester
+ present access_requester, with: Entities::AccessRequester
else
render_validation_error!(access_requester)
end
@@ -56,7 +56,7 @@ module API
member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute
status :created
- present member.user, with: Entities::Member, member: member
+ present member, with: Entities::Member
end
desc 'Denies an access request for the given user.' do
diff --git a/lib/api/api.rb b/lib/api/api.rb
index ae161efb358..f3f64244589 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -106,6 +106,7 @@ module API
# Keep in alphabetical order
mount ::API::AccessRequests
+ mount ::API::Applications
mount ::API::AwardEmoji
mount ::API::Boards
mount ::API::Branches
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 9aeebc34525..c2113551207 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -42,7 +42,7 @@ module API
include Gitlab::Auth::UserAuthFinders
def find_current_user!
- user = find_user_from_access_token || find_user_from_warden
+ user = find_user_from_sources
return unless user
forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api)
@@ -50,6 +50,10 @@ module API
user
end
+ def find_user_from_sources
+ find_user_from_access_token || find_user_from_warden
+ end
+
private
# An array of scopes that were registered (using `allow_access_with_scope`)
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
new file mode 100644
index 00000000000..b122cdefe4e
--- /dev/null
+++ b/lib/api/applications.rb
@@ -0,0 +1,27 @@
+module API
+ # External applications API
+ class Applications < Grape::API
+ before { authenticated_as_admin! }
+
+ resource :applications do
+ desc 'Create a new application' do
+ detail 'This feature was introduced in GitLab 10.5'
+ success Entities::ApplicationWithSecret
+ end
+ params do
+ requires :name, type: String, desc: 'Application name'
+ requires :redirect_uri, type: String, desc: 'Application redirect URI'
+ requires :scopes, type: String, desc: 'Application scopes'
+ end
+ post do
+ application = Doorkeeper::Application.new(declared_params)
+
+ if application.save
+ present application, with: Entities::ApplicationWithSecret
+ else
+ render_validation_error! application
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 0791a110c39..1794207e29b 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -29,6 +29,8 @@ module API
use :pagination
end
get ':id/repository/branches' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329')
+
repository = user_project.repository
branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name))
merged_branch_names = repository.merged_branch_names(branches.map(&:name))
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 3f4b62dc1b2..e13463ec66b 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -205,22 +205,15 @@ module API
expose :build_artifacts_size, as: :job_artifacts_size
end
- class Member < UserBasic
- expose :access_level do |user, options|
- member = options[:member] || options[:source].members.find_by(user_id: user.id)
- member.access_level
- end
- expose :expires_at do |user, options|
- member = options[:member] || options[:source].members.find_by(user_id: user.id)
- member.expires_at
- end
+ class Member < Grape::Entity
+ expose :user, merge: true, using: UserBasic
+ expose :access_level
+ expose :expires_at
end
- class AccessRequester < UserBasic
- expose :requested_at do |user, options|
- access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id)
- access_requester.requested_at
- end
+ class AccessRequester < Grape::Entity
+ expose :user, merge: true, using: UserBasic
+ expose :requested_at
end
class Group < Grape::Entity
@@ -507,7 +500,15 @@ module API
expose :work_in_progress?, as: :work_in_progress
expose :milestone, using: Entities::Milestone
expose :merge_when_pipeline_succeeds
- expose :merge_status
+
+ # Ideally we should deprecate `MergeRequest#merge_status` exposure and
+ # use `MergeRequest#mergeable?` instead (boolean).
+ # See https://gitlab.com/gitlab-org/gitlab-ce/issues/42344 for more
+ # information.
+ expose :merge_status do |merge_request|
+ merge_request.check_if_can_be_merged
+ merge_request.merge_status
+ end
expose :diff_head_sha, as: :sha
expose :merge_commit_sha
expose :user_notes_count
@@ -1157,5 +1158,15 @@ module API
pages_domain
end
end
+
+ class Application < Grape::Entity
+ expose :uid, as: :application_id
+ expose :redirect_uri, as: :callback_url
+ end
+
+ # Use with care, this exposes the secret
+ class ApplicationWithSecret < Application
+ expose :secret
+ end
end
end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index eff1c5b70ea..eb67de81a0d 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -1,11 +1,6 @@
module API
module Helpers
module InternalHelpers
- SSH_GITALY_FEATURES = {
- 'git-receive-pack' => [:ssh_receive_pack, Gitlab::GitalyClient::MigrationStatus::OPT_IN],
- 'git-upload-pack' => [:ssh_upload_pack, Gitlab::GitalyClient::MigrationStatus::OPT_OUT]
- }.freeze
-
attr_reader :redirected_path
def wiki?
@@ -102,8 +97,14 @@ module API
# Return the Gitaly Address if it is enabled
def gitaly_payload(action)
- feature, status = SSH_GITALY_FEATURES[action]
- return unless feature && Gitlab::GitalyClient.feature_enabled?(feature, status: status)
+ return unless %w[git-receive-pack git-upload-pack].include?(action)
+
+ if action == 'git-receive-pack'
+ return unless Gitlab::GitalyClient.feature_enabled?(
+ :ssh_receive_pack,
+ status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT
+ )
+ end
{
repository: repository.gitaly_repository,
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 2cae53dba53..3d0d1287407 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -1,15 +1,13 @@
module API
module Helpers
module Runner
- include Gitlab::CurrentSettings
-
JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
JOB_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 10 * 60
def runner_registration_token_valid?
ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
- current_application_settings.runners_registration_token)
+ Gitlab::CurrentSettings.runners_registration_token)
end
def get_runner_version_from_params
@@ -70,7 +68,7 @@ module API
end
def max_artifacts_size
- current_application_settings.max_artifacts_size.megabytes.to_i
+ Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i
end
end
end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index c99fe3ab5b3..b6c278c89d0 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -161,6 +161,8 @@ module API
use :issue_params
end
post ':id/issues' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320')
+
authorize! :create_issue, user_project
# Setting created_at time only allowed for admins and project owners
@@ -201,6 +203,8 @@ module API
:labels, :created_at, :due_date, :confidential, :state_event
end
put ':id/issues/:issue_iid' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322')
+
issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
authorize! :update_issue, issue
@@ -234,6 +238,8 @@ module API
requires :to_project_id, type: Integer, desc: 'The ID of the new project'
end
post ':id/issues/:issue_iid/move' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323')
+
issue = user_project.issues.find_by(iid: params[:issue_iid])
not_found!('Issue') unless issue
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 5446f6b54b1..bc1de37284a 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -21,10 +21,11 @@ module API
get ":id/members" do
source = find_source(source_type, params[:id])
- users = source.users
- users = users.merge(User.search(params[:query])) if params[:query]
+ members = source.members.where.not(user_id: nil).includes(:user)
+ members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present?
+ members = paginate(members)
- present paginate(users), with: Entities::Member, source: source
+ present members, with: Entities::Member
end
desc 'Gets a member of a group or project.' do
@@ -39,7 +40,7 @@ module API
members = source.members
member = members.find_by!(user_id: params[:user_id])
- present member.user, with: Entities::Member, member: member
+ present member, with: Entities::Member
end
desc 'Adds a member to a group or project.' do
@@ -62,7 +63,7 @@ module API
if !member
not_allowed! # This currently can only be reached in EE
elsif member.persisted? && member.valid?
- present member.user, with: Entities::Member, member: member
+ present member, with: Entities::Member
else
render_validation_error!(member)
end
@@ -83,7 +84,7 @@ module API
member = source.members.find_by!(user_id: params.delete(:user_id))
if member.update_attributes(declared_params(include_missing: false))
- present member.user, with: Entities::Member, member: member
+ present member, with: Entities::Member
else
render_validation_error!(member)
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 420aaf1c964..719afa09295 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -152,6 +152,8 @@ module API
use :optional_params
end
post ":id/merge_requests" do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316')
+
authorize! :create_merge_request, user_project
mr_params = declared_params(include_missing: false)
@@ -256,6 +258,8 @@ module API
at_least_one_of(*at_least_one_of_ce)
end
put ':id/merge_requests/:merge_request_iid' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318')
+
merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
mr_params = declared_params(include_missing: false)
@@ -283,6 +287,8 @@ module API
optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch'
end
put ':id/merge_requests/:merge_request_iid/merge' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317')
+
merge_request = find_project_merge_request(params[:merge_request_iid])
merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds])
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 675c963bae2..d2b8b832e4e 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -42,6 +42,8 @@ module API
requires :ref, type: String, desc: 'Reference'
end
post ':id/pipeline' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124')
+
authorize! :create_pipeline, user_project
new_pipeline = Ci::CreatePipelineService.new(user_project,
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 8b5e4f8edcc..5b481121a10 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -210,6 +210,8 @@ module API
optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into'
end
post ':id/fork' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42284')
+
fork_params = declared_params(include_missing: false)
namespace_id = fork_params[:namespace]
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 80feb629d54..1f80646a2ea 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -215,9 +215,9 @@ module API
job = authenticate_job!
forbidden!('Job is not running!') unless job.running?
- artifacts_upload_path = JobArtifactUploader.artifacts_upload_path
- artifacts = uploaded_file(:file, artifacts_upload_path)
- metadata = uploaded_file(:metadata, artifacts_upload_path)
+ workhorse_upload_path = JobArtifactUploader.workhorse_upload_path
+ artifacts = uploaded_file(:file, workhorse_upload_path)
+ metadata = uploaded_file(:metadata, workhorse_upload_path)
bad_request!('Missing artifacts file!') unless artifacts
file_to_large! unless artifacts.size < max_artifacts_size
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 6550b331fb8..41862768a3f 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -17,15 +17,15 @@ module API
}
}.freeze
PROJECT_TEMPLATE_REGEX =
- /[\<\{\[]
+ %r{[\<\{\[]
(project|description|
one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
- [\>\}\]]/xi.freeze
+ [\>\}\]]}xi.freeze
YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
FULLNAME_TEMPLATE_REGEX =
- /[\<\{\[]
+ %r{[\<\{\[]
(fullname|name\sof\s(author|copyright\sowner))
- [\>\}\]]/xi.freeze
+ [\>\}\]]}xi.freeze
helpers do
def parsed_license_template
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index dd6801664b1..b3709455bc3 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -15,6 +15,8 @@ module API
optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
end
post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42283')
+
# validate variables
params[:variables] = params[:variables].to_h
unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) }
diff --git a/lib/api/users.rb b/lib/api/users.rb
index e5de31ad51b..c7c2aa280d5 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -383,6 +383,8 @@ module API
optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions"
end
delete ":id" do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279')
+
authenticated_as_admin!
user = User.find_by(id: params[:id])
diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb
index b201bf77667..25176c5b38e 100644
--- a/lib/api/v3/branches.rb
+++ b/lib/api/v3/branches.rb
@@ -14,6 +14,8 @@ module API
success ::API::Entities::Branch
end
get ":id/repository/branches" do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42276')
+
repository = user_project.repository
branches = repository.branches.sort_by(&:name)
merged_branch_names = repository.merged_branch_names(branches.map(&:name))
diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb
index cb371fdbab8..b59947d81d9 100644
--- a/lib/api/v3/issues.rb
+++ b/lib/api/v3/issues.rb
@@ -134,6 +134,8 @@ module API
use :issue_params
end
post ':id/issues' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42131')
+
# Setting created_at time only allowed for admins and project owners
unless current_user.admin? || user_project.owner == current_user
params.delete(:created_at)
@@ -169,6 +171,8 @@ module API
:labels, :created_at, :due_date, :confidential, :state_event
end
put ':id/issues/:issue_id' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42132')
+
issue = user_project.issues.find(params.delete(:issue_id))
authorize! :update_issue, issue
@@ -201,6 +205,8 @@ module API
requires :to_project_id, type: Integer, desc: 'The ID of the new project'
end
post ':id/issues/:issue_id/move' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42133')
+
issue = user_project.issues.find_by(id: params[:issue_id])
not_found!('Issue') unless issue
diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb
index de226e4e573..d7bde8ceb89 100644
--- a/lib/api/v3/members.rb
+++ b/lib/api/v3/members.rb
@@ -22,10 +22,11 @@ module API
get ":id/members" do
source = find_source(source_type, params[:id])
- users = source.users
- users = users.merge(User.search(params[:query])) if params[:query]
+ members = source.members.where.not(user_id: nil).includes(:user)
+ members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present?
+ members = paginate(members)
- present paginate(users), with: ::API::Entities::Member, source: source
+ present members, with: ::API::Entities::Member
end
desc 'Gets a member of a group or project.' do
@@ -40,7 +41,7 @@ module API
members = source.members
member = members.find_by!(user_id: params[:user_id])
- present member.user, with: ::API::Entities::Member, member: member
+ present member, with: ::API::Entities::Member
end
desc 'Adds a member to a group or project.' do
@@ -69,7 +70,7 @@ module API
end
if member.persisted? && member.valid?
- present member.user, with: ::API::Entities::Member, member: member
+ present member, with: ::API::Entities::Member
else
# This is to ensure back-compatibility but 400 behavior should be used
# for all validation errors in 9.0!
@@ -93,7 +94,7 @@ module API
member = source.members.find_by!(user_id: params.delete(:user_id))
if member.update_attributes(declared_params(include_missing: false))
- present member.user, with: ::API::Entities::Member, member: member
+ present member, with: ::API::Entities::Member
else
# This is to ensure back-compatibility but 400 behavior should be used
# for all validation errors in 9.0!
@@ -125,7 +126,7 @@ module API
else
::Members::DestroyService.new(source, current_user, declared_params).execute
- present member.user, with: ::API::Entities::Member, member: member
+ present member, with: ::API::Entities::Member
end
end
end
diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb
index 0a24fea52a3..ce216497996 100644
--- a/lib/api/v3/merge_requests.rb
+++ b/lib/api/v3/merge_requests.rb
@@ -91,6 +91,8 @@ module API
use :optional_params
end
post ":id/merge_requests" do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126')
+
authorize! :create_merge_request, user_project
mr_params = declared_params(include_missing: false)
@@ -167,6 +169,8 @@ module API
:remove_source_branch
end
put path do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42127')
+
merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request)
mr_params = declared_params(include_missing: false)
diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb
index c48cbd2b765..6d31c12f572 100644
--- a/lib/api/v3/pipelines.rb
+++ b/lib/api/v3/pipelines.rb
@@ -19,6 +19,8 @@ module API
desc: 'Either running, branches, or tags'
end
get ':id/pipelines' do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42123')
+
authorize! :read_pipeline, user_project
pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute
diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb
index 446f804124b..c856ba99f09 100644
--- a/lib/api/v3/projects.rb
+++ b/lib/api/v3/projects.rb
@@ -173,9 +173,9 @@ module API
use :sort_params
use :pagination
end
- get "/search/:query", requirements: { query: /[^\/]+/ } do
+ get "/search/:query", requirements: { query: %r{[^/]+} } do
search_service = Search::GlobalService.new(current_user, search: params[:query]).execute
- projects = search_service.objects('projects', params[:page])
+ projects = search_service.objects('projects', params[:page], false)
projects = projects.reorder(params[:order_by] => params[:sort])
present paginate(projects), with: ::API::V3::Entities::Project
diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb
index 7298203df10..b82b02b5f49 100644
--- a/lib/api/v3/templates.rb
+++ b/lib/api/v3/templates.rb
@@ -16,15 +16,15 @@ module API
}
}.freeze
PROJECT_TEMPLATE_REGEX =
- /[\<\{\[]
+ %r{[\<\{\[]
(project|description|
one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
- [\>\}\]]/xi.freeze
+ [\>\}\]]}xi.freeze
YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
FULLNAME_TEMPLATE_REGEX =
- /[\<\{\[]
+ %r{[\<\{\[]
(fullname|name\sof\s(author|copyright\sowner))
- [\>\}\]]/xi.freeze
+ [\>\}\]]}xi.freeze
DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze
helpers do
diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb
index 534911fde5c..34f07dfb486 100644
--- a/lib/api/v3/triggers.rb
+++ b/lib/api/v3/triggers.rb
@@ -16,6 +16,8 @@ module API
optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
end
post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do
+ Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42121')
+
# validate variables
params[:variables] = params[:variables].to_h
unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) }
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index 7a582a20056..4383124d150 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -3,7 +3,7 @@ require 'backup/files'
module Backup
class Artifacts < Files
def initialize
- super('artifacts', LegacyArtifactUploader.local_store_path)
+ super('artifacts', JobArtifactUploader.root)
end
def create_files_dir
diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb
new file mode 100644
index 00000000000..355c364b07b
--- /dev/null
+++ b/lib/banzai/color_parser.rb
@@ -0,0 +1,44 @@
+module Banzai
+ module ColorParser
+ ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0
+ PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100%
+ ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/
+ BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255
+ DEGS = /-?\d+(?:deg)?/i # [-]digits[deg]
+ RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad
+ HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/
+ RGB_FORMAT = %r{
+ (?:rgba?
+ \(
+ (?:
+ (?:(?:#{BITS},\s*){2}#{BITS})
+ |
+ (?:(?:#{PERCENTS},\s*){2}#{PERCENTS})
+ )
+ #{ALPHA_CHANNEL}
+ \)
+ )
+ }xi
+ HSL_FORMAT = %r{
+ (?:hsla?
+ \(
+ (?:#{DEGS}|#{RADS}),\s*#{PERCENTS},\s*#{PERCENTS}
+ #{ALPHA_CHANNEL}
+ \)
+ )
+ }xi
+
+ FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze
+
+ COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix
+
+ # Public: Analyzes whether the String is a color code.
+ #
+ # text - The String to be parsed.
+ #
+ # Returns the recognized color String or nil if none was found.
+ def self.parse(text)
+ text if COLOR_FORMAT =~ text
+ end
+ end
+end
diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb
new file mode 100644
index 00000000000..6ab29ac281f
--- /dev/null
+++ b/lib/banzai/filter/color_filter.rb
@@ -0,0 +1,31 @@
+module Banzai
+ module Filter
+ # HTML filter that renders `color` followed by a color "chip".
+ #
+ class ColorFilter < HTML::Pipeline::Filter
+ COLOR_CHIP_CLASS = 'gfm-color_chip'.freeze
+
+ def call
+ doc.css('code').each do |node|
+ color = ColorParser.parse(node.content)
+ node << color_chip(color) if color
+ end
+
+ doc
+ end
+
+ private
+
+ def color_chip(color)
+ checkerboard = doc.document.create_element('span', class: COLOR_CHIP_CLASS)
+ chip = doc.document.create_element('span', style: inline_styles(color: color))
+
+ checkerboard << chip
+ end
+
+ def inline_styles(color:)
+ "background-color: #{color};"
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb
index 6255a611dbe..b82c6ca6393 100644
--- a/lib/banzai/filter/emoji_filter.rb
+++ b/lib/banzai/filter/emoji_filter.rb
@@ -54,9 +54,9 @@ module Banzai
# Build a regexp that matches all valid :emoji: names.
def self.emoji_pattern
@emoji_pattern ||=
- /(?<=[^[:alnum:]:]|\n|^)
+ %r{(?<=[^[:alnum:]:]|\n|^)
:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):
- (?=[^[:alnum:]:]|$)/x
+ (?=[^[:alnum:]:]|$)}x
end
# Build a regexp that matches all valid unicode emojis names.
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index 2e259904673..c2b42673376 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -51,10 +51,10 @@ module Banzai
# See https://github.com/gollum/gollum/wiki
#
# Rubular: http://rubular.com/r/7dQnE5CUCH
- TAGS_PATTERN = %r{\[\[(.+?)\]\]}.freeze
+ TAGS_PATTERN = /\[\[(.+?)\]\]/.freeze
# Pattern to match allowed image extensions
- ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i.freeze
+ ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze
def call
search_text_nodes(doc).each do |node|
diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb
index adc09c8afbd..5dd572de3a1 100644
--- a/lib/banzai/pipeline/broadcast_message_pipeline.rb
+++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb
@@ -7,6 +7,7 @@ module Banzai
Filter::SanitizationFilter,
Filter::EmojiFilter,
+ Filter::ColorFilter,
Filter::AutolinkFilter,
Filter::ExternalLinkFilter
]
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index c746f6f64e9..4001b8a85e3 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -14,6 +14,7 @@ module Banzai
Filter::SyntaxHighlightFilter,
Filter::MathFilter,
+ Filter::ColorFilter,
Filter::MermaidFilter,
Filter::VideoLinkFilter,
Filter::ImageLazyLoadFilter,
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
index 63bce655f57..f90d711474a 100644
--- a/lib/container_registry/registry.rb
+++ b/lib/container_registry/registry.rb
@@ -11,7 +11,7 @@ module ContainerRegistry
private
def default_path
- @uri.sub(/^https?:\/\//, '')
+ @uri.sub(%r{^https?://}, '')
end
end
end
diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb
index f2bf3d0fb2b..3978a6d9fe4 100644
--- a/lib/email_template_interceptor.rb
+++ b/lib/email_template_interceptor.rb
@@ -1,10 +1,8 @@
# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails
class EmailTemplateInterceptor
- extend Gitlab::CurrentSettings
-
def self.delivering_email(message)
# Remove HTML part if HTML emails are disabled.
- unless current_application_settings.html_emails_enabled
+ unless Gitlab::CurrentSettings.html_emails_enabled
message.parts.delete_if do |part|
part.content_type.start_with?('text/html')
end
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index d8aca3304c5..a9b04c183ad 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -56,7 +56,7 @@ module ExtractsPath
if valid_refs.length == 0
# No exact ref match, so just try our best
- pair = id.match(/([^\/]+)(.*)/).captures
+ pair = id.match(%r{([^/]+)(.*)}).captures
else
# There is a distinct possibility that multiple refs prefix the ID.
# Use the longest match to maximize the chance that we have the
@@ -68,7 +68,7 @@ module ExtractsPath
end
# Remove ending slashes from path
- pair[1].gsub!(/^\/|\/$/, '')
+ pair[1].gsub!(%r{^/|/$}, '')
pair
end
diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb
new file mode 100644
index 00000000000..605e93022e7
--- /dev/null
+++ b/lib/gitaly/server.rb
@@ -0,0 +1,43 @@
+module Gitaly
+ class Server
+ def self.all
+ Gitlab.config.repositories.storages.keys.map { |s| Gitaly::Server.new(s) }
+ end
+
+ attr_reader :storage
+
+ def initialize(storage)
+ @storage = storage
+ end
+
+ def server_version
+ info.server_version
+ end
+
+ def git_binary_version
+ info.git_version
+ end
+
+ def up_to_date?
+ server_version == Gitlab::GitalyClient.expected_server_version
+ end
+
+ def address
+ Gitlab::GitalyClient.address(@storage)
+ rescue RuntimeError => e
+ "Error getting the address: #{e.message}"
+ end
+
+ private
+
+ def info
+ @info ||=
+ begin
+ Gitlab::GitalyClient::ServerService.new(@storage).info
+ rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded
+ # This will show the server as being out of date
+ Gitaly::ServerInfoResponse.new(git_version: '', server_version: '')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index cead1c7eacd..ee7f4be6b9f 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -6,8 +6,6 @@ module Gitlab
# Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
# the resulting HTML through HTML pipeline filters.
module Asciidoc
- extend Gitlab::CurrentSettings
-
DEFAULT_ADOC_ATTRS = [
'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab',
'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font'
@@ -33,9 +31,9 @@ module Gitlab
def self.plantuml_setup
Asciidoctor::PlantUml.configure do |conf|
- conf.url = current_application_settings.plantuml_url
- conf.svg_enable = current_application_settings.plantuml_enabled
- conf.png_enable = current_application_settings.plantuml_enabled
+ conf.url = Gitlab::CurrentSettings.plantuml_url
+ conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled
+ conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled
conf.txt_enable = false
end
end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 65d7fd3ec70..05932378173 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -14,8 +14,6 @@ module Gitlab
DEFAULT_SCOPES = [:api].freeze
class << self
- include Gitlab::CurrentSettings
-
def find_for_git_client(login, password, project:, ip:)
raise "Must provide an IP for rate limiting" if ip.nil?
@@ -57,7 +55,7 @@ module Gitlab
if user.nil? || user.ldap_user?
# Second chance - try LDAP authentication
Gitlab::LDAP::Authentication.login(login, password)
- elsif current_application_settings.password_authentication_enabled_for_git?
+ elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git?
user if user.active? && user.valid_password?(password)
end
end
@@ -87,7 +85,7 @@ module Gitlab
private
def authenticate_using_internal_or_ldap_password?
- current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled?
+ Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled?
end
def service_request_check(login, password, project)
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
index 81e95e5832d..8a8e770940e 100644
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -12,7 +12,7 @@ module Gitlab
# Ends with /:random_hex/:filename
FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
- FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}}
+ FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/
# These regex patterns are tested against a relative path, relative to
# the upload directory.
@@ -143,7 +143,7 @@ module Gitlab
end
def absolute_path
- File.join(CarrierWave.root, path)
+ File.join(Gitlab.config.uploads.storage_path, path)
end
end
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
index 4e0121ca34d..a7a1bbe1752 100644
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
@@ -11,9 +11,12 @@ module Gitlab
FIND_BATCH_SIZE = 500
RELATIVE_UPLOAD_DIR = "uploads".freeze
- ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze
+ ABSOLUTE_UPLOAD_DIR = File.join(
+ Gitlab.config.uploads.storage_path,
+ RELATIVE_UPLOAD_DIR
+ )
FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
- START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/}
+ START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}
EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze
EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze
@@ -81,7 +84,7 @@ module Gitlab
paths = []
stdout.each_line("\0") do |line|
- paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '')
+ paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
if paths.size >= batch_size
yield(paths)
diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb
index 9a0482306b7..778d78185ff 100644
--- a/lib/gitlab/badge/coverage/report.rb
+++ b/lib/gitlab/badge/coverage/report.rb
@@ -23,7 +23,7 @@ module Gitlab
@coverage ||= raw_coverage
return unless @coverage
- @coverage.to_i
+ @coverage.to_f.round(2)
end
def metadata
diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb
index fcecb1d9665..afbf9dd17e3 100644
--- a/lib/gitlab/badge/coverage/template.rb
+++ b/lib/gitlab/badge/coverage/template.rb
@@ -25,7 +25,7 @@ module Gitlab
end
def value_text
- @status ? "#{@status}%" : 'unknown'
+ @status ? ("%.2f%%" % @status) : 'unknown'
end
def key_width
@@ -33,7 +33,7 @@ module Gitlab
end
def value_width
- @status ? 36 : 58
+ @status ? 54 : 58
end
def value_color
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
index c0c666dfb7b..fe267248275 100644
--- a/lib/gitlab/bare_repository_import/repository.rb
+++ b/lib/gitlab/bare_repository_import/repository.rb
@@ -1,3 +1,5 @@
+# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953
+#
module Gitlab
module BareRepositoryImport
class Repository
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index ef92fc5a0a0..945d70e7a24 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -16,7 +16,7 @@ module Gitlab
lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'
}.freeze
- attr_reader :user_access, :project, :skip_authorization, :protocol
+ attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name
def initialize(
change, user_access:, project:, skip_authorization: false,
@@ -51,9 +51,9 @@ module Gitlab
end
def branch_checks
- return unless @branch_name
+ return unless branch_name
- if deletion? && @branch_name == project.default_branch
+ if deletion? && branch_name == project.default_branch
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
end
@@ -61,7 +61,7 @@ module Gitlab
end
def protected_branch_checks
- return unless ProtectedBranch.protected?(project, @branch_name)
+ return unless ProtectedBranch.protected?(project, branch_name)
if forced_push?
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
@@ -75,29 +75,29 @@ module Gitlab
end
def protected_branch_deletion_checks
- unless user_access.can_delete_branch?(@branch_name)
+ unless user_access.can_delete_branch?(branch_name)
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
end
- unless protocol == 'web'
+ unless updated_from_web?
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
end
end
def protected_branch_push_checks
if matching_merge_request?
- unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name)
+ unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name)
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
end
else
- unless user_access.can_push_to_branch?(@branch_name)
+ unless user_access.can_push_to_branch?(branch_name)
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch]
end
end
end
def tag_checks
- return unless @tag_name
+ return unless tag_name
if tag_exists? && user_access.cannot_do_action?(:admin_project)
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
@@ -107,40 +107,44 @@ module Gitlab
end
def protected_tag_checks
- return unless ProtectedTag.protected?(project, @tag_name)
+ return unless ProtectedTag.protected?(project, tag_name)
raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
- unless user_access.can_create_tag?(@tag_name)
+ unless user_access.can_create_tag?(tag_name)
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
end
end
private
+ def updated_from_web?
+ protocol == 'web'
+ end
+
def tag_exists?
- project.repository.tag_exists?(@tag_name)
+ project.repository.tag_exists?(tag_name)
end
def forced_push?
- Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev)
+ Gitlab::Checks::ForcePush.force_push?(project, oldrev, newrev)
end
def update?
- !Gitlab::Git.blank_ref?(@oldrev) && !deletion?
+ !Gitlab::Git.blank_ref?(oldrev) && !deletion?
end
def deletion?
- Gitlab::Git.blank_ref?(@newrev)
+ Gitlab::Git.blank_ref?(newrev)
end
def matching_merge_request?
- Checks::MatchingMergeRequest.new(@newrev, @branch_name, @project).match?
+ Checks::MatchingMergeRequest.new(newrev, branch_name, project).match?
end
def lfs_objects_exist_check
- lfs_check = Checks::LfsIntegrity.new(project, @newrev)
+ lfs_check = Checks::LfsIntegrity.new(project, newrev)
if lfs_check.objects_missing?
raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing]
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 5b2f09e03ea..428c0505808 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -97,7 +97,7 @@ module Gitlab
end
def total_size
- descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}}
+ descendant_pattern = /^#{Regexp.escape(@path.to_s)}/
entries.sum do |path, entry|
(entry[:size] if path =~ descendant_pattern).to_i
end
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index baf55b1fa07..f2e5124c8a8 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -52,12 +52,14 @@ module Gitlab
end
def exist?
- current_path.present? || old_trace.present?
+ trace_artifact&.exists? || current_path.present? || old_trace.present?
end
def read
stream = Gitlab::Ci::Trace::Stream.new do
- if current_path
+ if trace_artifact
+ trace_artifact.open
+ elsif current_path
File.open(current_path, "rb")
elsif old_trace
StringIO.new(old_trace)
@@ -82,6 +84,8 @@ module Gitlab
end
def erase!
+ trace_artifact&.destroy
+
paths.each do |trace_path|
FileUtils.rm(trace_path, force: true)
end
@@ -137,6 +141,10 @@ module Gitlab
"#{job.id}.log"
) if job.project&.ci_id
end
+
+ def trace_artifact
+ job.job_artifacts_trace
+ end
end
end
end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 91fd9cc7631..b7c596a973d 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -1,73 +1,79 @@
module Gitlab
module CurrentSettings
- extend self
+ class << self
+ def current_application_settings
+ if RequestStore.active?
+ RequestStore.fetch(:current_application_settings) { ensure_application_settings! }
+ else
+ ensure_application_settings!
+ end
+ end
- def current_application_settings
- if RequestStore.active?
- RequestStore.fetch(:current_application_settings) { ensure_application_settings! }
- else
- ensure_application_settings!
+ def fake_application_settings(defaults = ::ApplicationSetting.defaults)
+ Gitlab::FakeApplicationSettings.new(defaults)
end
- end
- delegate :sidekiq_throttling_enabled?, to: :current_application_settings
+ def method_missing(name, *args, &block)
+ current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend
+ end
- def fake_application_settings(defaults = ::ApplicationSetting.defaults)
- FakeApplicationSettings.new(defaults)
- end
+ def respond_to_missing?(name, include_private = false)
+ current_application_settings.respond_to?(name, include_private) || super
+ end
- private
+ private
- def ensure_application_settings!
- return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true'
+ def ensure_application_settings!
+ return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true'
- cached_application_settings || uncached_application_settings
- end
+ cached_application_settings || uncached_application_settings
+ end
- def cached_application_settings
- begin
- ::ApplicationSetting.cached
- rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
- # In case Redis isn't running or the Redis UNIX socket file is not available
+ def cached_application_settings
+ begin
+ ::ApplicationSetting.cached
+ rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
+ # In case Redis isn't running or the Redis UNIX socket file is not available
+ end
end
- end
- def uncached_application_settings
- return fake_application_settings unless connect_to_db?
+ def uncached_application_settings
+ return fake_application_settings unless connect_to_db?
- db_settings = ::ApplicationSetting.current
+ db_settings = ::ApplicationSetting.current
- # If there are pending migrations, it's possible there are columns that
- # need to be added to the application settings. To prevent Rake tasks
- # and other callers from failing, use any loaded settings and return
- # defaults for missing columns.
- if ActiveRecord::Migrator.needs_migration?
- defaults = ::ApplicationSetting.defaults
- defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present?
- return fake_application_settings(defaults)
- end
+ # If there are pending migrations, it's possible there are columns that
+ # need to be added to the application settings. To prevent Rake tasks
+ # and other callers from failing, use any loaded settings and return
+ # defaults for missing columns.
+ if ActiveRecord::Migrator.needs_migration?
+ defaults = ::ApplicationSetting.defaults
+ defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present?
+ return fake_application_settings(defaults)
+ end
- return db_settings if db_settings.present?
+ return db_settings if db_settings.present?
- ::ApplicationSetting.create_from_defaults || in_memory_application_settings
- end
+ ::ApplicationSetting.create_from_defaults || in_memory_application_settings
+ end
- def in_memory_application_settings
- @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables
- rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError
- # In case migrations the application_settings table is not created yet,
- # we fallback to a simple OpenStruct
- fake_application_settings
- end
+ def in_memory_application_settings
+ @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError
+ # In case migrations the application_settings table is not created yet,
+ # we fallback to a simple OpenStruct
+ fake_application_settings
+ end
- def connect_to_db?
- # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised
- active_db_connection = ActiveRecord::Base.connection.active? rescue false
+ def connect_to_db?
+ # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised
+ active_db_connection = ActiveRecord::Base.connection.active? rescue false
- active_db_connection &&
- ActiveRecord::Base.connection.table_exists?('application_settings')
- rescue ActiveRecord::NoDatabaseError
- false
+ active_db_connection &&
+ ActiveRecord::Base.connection.table_exists?('application_settings')
+ rescue ActiveRecord::NoDatabaseError
+ false
+ end
end
end
end
diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb
index 9f76967fc77..d32837f5793 100644
--- a/lib/gitlab/database/grant.rb
+++ b/lib/gitlab/database/grant.rb
@@ -12,30 +12,40 @@ module Gitlab
# Returns true if the current user can create and execute triggers on the
# given table.
def self.create_and_execute_trigger?(table)
- priv =
- if Database.postgresql?
- where(privilege_type: 'TRIGGER', table_name: table)
- .where('grantee = user')
- else
- queries = [
- Grant.select(1)
- .from('information_schema.user_privileges')
- .where("PRIVILEGE_TYPE = 'SUPER'")
- .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"),
+ if Database.postgresql?
+ # We _must not_ use quote_table_name as this will produce double
+ # quotes on PostgreSQL and for "has_table_privilege" we need single
+ # quotes.
+ quoted_table = connection.quote(table)
- Grant.select(1)
- .from('information_schema.schema_privileges')
- .where("PRIVILEGE_TYPE = 'TRIGGER'")
- .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name)
- .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
- ]
+ begin
+ from(nil)
+ .pluck("has_table_privilege(#{quoted_table}, 'TRIGGER')")
+ .first
+ rescue ActiveRecord::StatementInvalid
+ # This error is raised when using a non-existing table name. In this
+ # case we just want to return false as a user technically can't
+ # create triggers for such a table.
+ false
+ end
+ else
+ queries = [
+ Grant.select(1)
+ .from('information_schema.user_privileges')
+ .where("PRIVILEGE_TYPE = 'SUPER'")
+ .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')"),
- union = SQL::Union.new(queries).to_sql
+ Grant.select(1)
+ .from('information_schema.schema_privileges')
+ .where("PRIVILEGE_TYPE = 'TRIGGER'")
+ .where('TABLE_SCHEMA = ?', Gitlab::Database.database_name)
+ .where("GRANTEE = CONCAT('\\'', REPLACE(CURRENT_USER(), '@', '\\'@\\''), '\\'')")
+ ]
- Grant.from("(#{union}) privs")
- end
+ union = SQL::Union.new(queries).to_sql
- priv.any?
+ Grant.from("(#{union}) privs").any?
+ end
end
end
end
diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb
index 0245bf4077a..cfd4ec15125 100644
--- a/lib/gitlab/dependency_linker/composer_json_linker.rb
+++ b/lib/gitlab/dependency_linker/composer_json_linker.rb
@@ -11,7 +11,7 @@ module Gitlab
end
def package_url(name)
- "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z}
+ "https://packagist.org/packages/#{name}" if name =~ /\A#{REPO_REGEX}\z/
end
end
end
diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb
index d034ea67387..bfea836bcb2 100644
--- a/lib/gitlab/dependency_linker/gemfile_linker.rb
+++ b/lib/gitlab/dependency_linker/gemfile_linker.rb
@@ -15,7 +15,7 @@ module Gitlab
link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url))
# Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo
- link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+ link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself)
# Link `source "https://rubygems.org"` to https://rubygems.org
link_method_call('source', URL_REGEX, &:itself)
diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb
index a52c7a02439..924e55e9820 100644
--- a/lib/gitlab/dependency_linker/podspec_linker.rb
+++ b/lib/gitlab/dependency_linker/podspec_linker.rb
@@ -12,7 +12,7 @@ module Gitlab
def link_dependencies
link_method_call('homepage', URL_REGEX, &:itself)
- link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself)
+ link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself)
link_method_call('license', &method(:license_url))
link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url))
diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb
index d3b49b1ec75..0fb71976883 100644
--- a/lib/gitlab/ee_compat_check.rb
+++ b/lib/gitlab/ee_compat_check.rb
@@ -5,7 +5,7 @@ module Gitlab
DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
CHECK_DIR = Rails.root.join('ee_compat_check')
- IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze
+ IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze
PLEASE_READ_THIS_BANNER = %Q{
============================================================
===================== PLEASE READ THIS =====================
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 558df87f36d..01c28d051ee 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -43,7 +43,7 @@ module Gitlab
return "" unless decoded
# Certain trigger phrases that means we didn't parse correctly
- if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+ if decoded =~ %r{(Content\-Type\:|multipart/alternative|text/plain)}
return ""
end
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index 0e9ef4f897c..cc2638172ec 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -6,14 +6,14 @@ module Gitlab
module FileDetector
PATTERNS = {
# Project files
- readme: /\Areadme[^\/]*\z/i,
- changelog: /\A(changelog|history|changes|news)[^\/]*\z/i,
- license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i,
- contributing: /\Acontributing[^\/]*\z/i,
+ readme: %r{\Areadme[^/]*\z}i,
+ changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i,
+ license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i,
+ contributing: %r{\Acontributing[^/]*\z}i,
version: 'version',
avatar: /\Alogo\.(png|jpg|gif)\z/,
- issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/,
- merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/,
+ issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z},
+ merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z},
# Configuration files
gitignore: '.gitignore',
@@ -22,17 +22,17 @@ module Gitlab
route_map: '.gitlab/route-map.yml',
# Dependency files
- cartfile: /\ACartfile[^\/]*\z/,
+ cartfile: %r{\ACartfile[^/]*\z},
composer_json: 'composer.json',
gemfile: /\A(Gemfile|gems\.rb)\z/,
gemfile_lock: 'Gemfile.lock',
- gemspec: /\A[^\/]*\.gemspec\z/,
+ gemspec: %r{\A[^/]*\.gemspec\z},
godeps_json: 'Godeps.json',
package_json: 'package.json',
podfile: 'Podfile',
- podspec_json: /\A[^\/]*\.podspec\.json\z/,
- podspec: /\A[^\/]*\.podspec\z/,
- requirements_txt: /\A[^\/]*requirements\.txt\z/,
+ podspec_json: %r{\A[^/]*\.podspec\.json\z},
+ podspec: %r{\A[^/]*\.podspec\z},
+ requirements_txt: %r{\A[^/]*requirements\.txt\z},
yarn_lock: 'yarn.lock'
}.freeze
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index 8fab5489616..1b74f735679 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -27,7 +27,7 @@ module Gitlab
with_link_in_tmp_dir(file.file) do |open_tmp_file|
new_uploader.store!(open_tmp_file)
end
- new_uploader.to_markdown
+ new_uploader.markdown_link
end
end
@@ -46,7 +46,7 @@ module Gitlab
private
def find_file(project, secret, file)
- uploader = FileUploader.new(project, secret)
+ uploader = FileUploader.new(project, secret: secret)
uploader.retrieve_from_store!(file)
uploader.file
end
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 71647099f83..d4e893b881c 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -6,12 +6,13 @@ module Gitlab
CommandError = Class.new(StandardError)
CommitError = Class.new(StandardError)
+ OSError = Class.new(StandardError)
class << self
include Gitlab::EncodingHelper
def ref_name(ref)
- encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '')
+ encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '')
end
def branch_name(ref)
diff --git a/lib/gitlab/git/attributes_at_ref_parser.rb b/lib/gitlab/git/attributes_at_ref_parser.rb
new file mode 100644
index 00000000000..26b5bd520d5
--- /dev/null
+++ b/lib/gitlab/git/attributes_at_ref_parser.rb
@@ -0,0 +1,14 @@
+module Gitlab
+ module Git
+ # Parses root .gitattributes file at a given ref
+ class AttributesAtRefParser
+ delegate :attributes, to: :@parser
+
+ def initialize(repository, ref)
+ blob = repository.blob_at(ref, '.gitattributes')
+
+ @parser = AttributesParser.new(blob&.data)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes_parser.rb
index 2d20cd473a7..d8aeabb6cba 100644
--- a/lib/gitlab/git/attributes.rb
+++ b/lib/gitlab/git/attributes_parser.rb
@@ -1,42 +1,26 @@
-# Gitaly note: JV: not sure what to make of this class. Why does it use
-# the full disk path of the repository to look up attributes This is
-# problematic in Gitaly, because Gitaly hides the full disk path to the
-# repository from gitlab-ce.
-
module Gitlab
module Git
# Class for parsing Git attribute files and extracting the attributes for
# file patterns.
- #
- # Unlike Rugged this parser only needs a single IO call (a call to `open`),
- # vastly reducing the time spent in extracting attributes.
- #
- # This class _only_ supports parsing the attributes file located at
- # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files
- # (`.gitattributes` is copied to this particular path).
- #
- # Basic usage:
- #
- # attributes = Gitlab::Git::Attributes.new(some_repo.path)
- #
- # attributes.attributes('README.md') # => { "eol" => "lf }
- class Attributes
- # path - The path to the Git repository.
- def initialize(path)
- @path = File.expand_path(path)
- @patterns = nil
+ class AttributesParser
+ def initialize(attributes_data)
+ @data = attributes_data || ""
+
+ if @data.is_a?(File)
+ @patterns = parse_file
+ end
end
# Returns all the Git attributes for the given path.
#
- # path - A path to a file for which to get the attributes.
+ # file_path - A path to a file for which to get the attributes.
#
# Returns a Hash.
- def attributes(path)
- full_path = File.join(@path, path)
+ def attributes(file_path)
+ absolute_path = File.join('/', file_path)
patterns.each do |pattern, attrs|
- return attrs if File.fnmatch?(pattern, full_path)
+ return attrs if File.fnmatch?(pattern, absolute_path)
end
{}
@@ -98,16 +82,10 @@ module Gitlab
# Iterates over every line in the attributes file.
def each_line
- full_path = File.join(@path, 'info/attributes')
+ @data.each_line do |line|
+ break unless line.valid_encoding?
- return unless File.exist?(full_path)
-
- File.open(full_path, 'r') do |handle|
- handle.each_line do |line|
- break unless line.valid_encoding?
-
- yield line.strip
- end
+ yield line.strip
end
end
@@ -125,7 +103,8 @@ module Gitlab
parsed = attrs ? parse_attributes(attrs) : {}
- pairs << [File.join(@path, pattern), parsed]
+ absolute_pattern = File.join('/', pattern)
+ pairs << [absolute_pattern, parsed]
end
# Newer entries take precedence over older entries.
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 31effdba292..6d6ed065f79 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -42,9 +42,7 @@ module Gitlab
end
def load_blame_by_shelling_out
- cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
- # Read in binary mode to ensure ASCII-8BIT
- IO.popen(cmd, 'rb') {|io| io.read }
+ @repo.shell_blame(@sha, @path)
end
def process_raw_blame(output)
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 031fccba92b..4828301dbb9 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -34,7 +34,7 @@ module Gitlab
def raw(repository, sha)
Gitlab::GitalyClient.migrate(:git_blob_raw) do |is_enabled|
if is_enabled
- Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
+ repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE)
else
rugged_raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE)
end
@@ -70,11 +70,17 @@ module Gitlab
# Returns array of Gitlab::Git::Blob
# Does not guarantee blob data will be set
def batch_lfs_pointers(repository, blob_ids)
- blob_ids.lazy
- .select { |sha| possible_lfs_blob?(repository, sha) }
- .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
- .select(&:lfs_pointer?)
- .force
+ repository.gitaly_migrate(:batch_lfs_pointers) do |is_enabled|
+ if is_enabled
+ repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a)
+ else
+ blob_ids.lazy
+ .select { |sha| possible_lfs_blob?(repository, sha) }
+ .map { |sha| rugged_raw(repository, sha, limit: LFS_POINTER_MAX_SIZE) }
+ .select(&:lfs_pointer?)
+ .force
+ end
+ end
end
def binary?(data)
@@ -101,7 +107,7 @@ module Gitlab
def find_entry_by_path(repository, root_id, path)
root_tree = repository.lookup(root_id)
# Strip leading slashes
- path[/^\/*/] = ''
+ path[%r{^/*}] = ''
path_arr = path.split('/')
entry = root_tree.find do |entry|
@@ -132,7 +138,9 @@ module Gitlab
end
def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
- path = path.sub(/\A\/*/, '')
+ return unless path
+
+ path = path.sub(%r{\A/*}, '')
path = '/' if path.empty?
name = File.basename(path)
@@ -173,6 +181,8 @@ module Gitlab
end
def find_by_rugged(repository, sha, path, limit:)
+ return unless path
+
rugged_commit = repository.lookup(sha)
root_tree = rugged_commit.tree
@@ -254,7 +264,7 @@ module Gitlab
Gitlab::GitalyClient.migrate(:git_blob_load_all_data) do |is_enabled|
@data = begin
if is_enabled
- Gitlab::GitalyClient::BlobService.new(repository).get_blob(oid: id, limit: -1).data
+ repository.gitaly_blob_client.get_blob(oid: id, limit: -1).data
else
repository.lookup(id).content
end
diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb
index 3487e099381..ae7e88f0503 100644
--- a/lib/gitlab/git/branch.rb
+++ b/lib/gitlab/git/branch.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: no RPC's here.
-
module Gitlab
module Git
class Branch < Ref
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index ca94b4baa59..a203587aec1 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -44,7 +44,7 @@ module Gitlab
# branch1...branch2) From the git documentation:
# "git diff A...B" is equivalent to "git diff
# $(git-merge-base A B) B"
- repo.merge_base_commit(head, base)
+ repo.merge_base(head, base)
end
options ||= {}
diff --git a/lib/gitlab/git/info_attributes.rb b/lib/gitlab/git/info_attributes.rb
new file mode 100644
index 00000000000..e79a440950b
--- /dev/null
+++ b/lib/gitlab/git/info_attributes.rb
@@ -0,0 +1,49 @@
+# Gitaly note: JV: not sure what to make of this class. Why does it use
+# the full disk path of the repository to look up attributes This is
+# problematic in Gitaly, because Gitaly hides the full disk path to the
+# repository from gitlab-ce.
+
+module Gitlab
+ module Git
+ # Parses gitattributes at `$GIT_DIR/info/attributes`
+ #
+ # Unlike Rugged this parser only needs a single IO call (a call to `open`),
+ # vastly reducing the time spent in extracting attributes.
+ #
+ # This class _only_ supports parsing the attributes file located at
+ # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files
+ # (`.gitattributes` is copied to this particular path).
+ #
+ # Basic usage:
+ #
+ # attributes = Gitlab::Git::InfoAttributes.new(some_repo.path)
+ #
+ # attributes.attributes('README.md') # => { "eol" => "lf }
+ class InfoAttributes
+ delegate :attributes, :patterns, to: :parser
+
+ # path - The path to the Git repository.
+ def initialize(path)
+ @repo_path = File.expand_path(path)
+ end
+
+ def parser
+ @parser ||= begin
+ if File.exist?(attributes_path)
+ File.open(attributes_path, 'r') do |file_handle|
+ AttributesParser.new(file_handle)
+ end
+ else
+ AttributesParser.new("")
+ end
+ end
+ end
+
+ private
+
+ def attributes_path
+ @attributes_path ||= File.join(@repo_path, 'info/attributes')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb
index 3fb0e2eed93..280def182d5 100644
--- a/lib/gitlab/git/operation_service.rb
+++ b/lib/gitlab/git/operation_service.rb
@@ -131,7 +131,10 @@ module Gitlab
oldrev = branch.target
- if oldrev == repository.merge_base(newrev, branch.target)
+ merge_base = repository.merge_base(newrev, branch.target)
+ raise Gitlab::Git::Repository::InvalidRef unless merge_base
+
+ if oldrev == merge_base
oldrev
else
raise Gitlab::Git::CommitError.new('Branch diverged')
diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb
index 42c80aabd0a..155cf52f050 100644
--- a/lib/gitlab/git/path_helper.rb
+++ b/lib/gitlab/git/path_helper.rb
@@ -6,7 +6,7 @@ module Gitlab
class << self
def normalize_path(filename)
# Strip all leading slashes so that //foo -> foo
- filename[/^\/*/] = ''
+ filename[%r{^/*}] = ''
# Expand relative paths (e.g. foo/../bar)
filename = Pathname.new(filename)
diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb
index 1ccca13ce2f..e0bd2bbe47b 100644
--- a/lib/gitlab/git/popen.rb
+++ b/lib/gitlab/git/popen.rb
@@ -19,6 +19,8 @@ module Gitlab
cmd_output = ""
cmd_status = 0
Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
+ stdout.set_encoding(Encoding::ASCII_8BIT)
+
yield(stdin) if block_given?
stdin.close
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index a3ba9475ad0..fa71a4e7ea7 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -23,7 +23,7 @@ module Gitlab
# Ex.
# Ref.extract_branch_name('refs/heads/master') #=> 'master'
def self.extract_branch_name(str)
- str.gsub(/\Arefs\/heads\//, '')
+ str.gsub(%r{\Arefs/heads/}, '')
end
# Gitaly: this method will probably be migrated indirectly via its call sites.
diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb
index 38e9d2a8554..ebe46722890 100644
--- a/lib/gitlab/git/remote_mirror.rb
+++ b/lib/gitlab/git/remote_mirror.rb
@@ -6,7 +6,23 @@ module Gitlab
@ref_name = ref_name
end
- def update(only_branches_matching: [], only_tags_matching: [])
+ def update(only_branches_matching: [])
+ @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled|
+ if is_enabled
+ gitaly_update(only_branches_matching)
+ else
+ rugged_update(only_branches_matching)
+ end
+ end
+ end
+
+ private
+
+ def gitaly_update(only_branches_matching)
+ @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching)
+ end
+
+ def rugged_update(only_branches_matching)
local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching)
remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching)
@@ -15,8 +31,8 @@ module Gitlab
delete_refs(local_branches, remote_branches)
- local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching)
- remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching)
+ local_tags = refs_obj(@repository.tags)
+ remote_tags = refs_obj(@repository.remote_tags(@ref_name))
updated_tags = changed_refs(local_tags, remote_tags)
@repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present?
@@ -24,8 +40,6 @@ module Gitlab
delete_refs(local_tags, remote_tags)
end
- private
-
def refs_obj(refs, only_refs_matching: [])
refs.each_with_object({}) do |ref, refs|
next if only_refs_matching.present? && !only_refs_matching.include?(ref.name)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 455c9902a0a..ab1362a3bb0 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -102,7 +102,7 @@ module Gitlab
)
@path = File.join(storage_path, @relative_path)
@name = @relative_path.split("/").last
- @attributes = Gitlab::Git::Attributes.new(path)
+ @attributes = Gitlab::Git::InfoAttributes.new(path)
end
def ==(other)
@@ -133,7 +133,7 @@ module Gitlab
end
def exists?
- Gitlab::GitalyClient.migrate(:repository_exists) do |enabled|
+ Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled|
if enabled
gitaly_repository_client.exists?
else
@@ -462,15 +462,18 @@ module Gitlab
path: nil,
follow: false,
skip_merges: false,
- disable_walk: false,
after: nil,
before: nil
}
options = default_options.merge(options)
- options[:limit] ||= 0
options[:offset] ||= 0
+ limit = options[:limit]
+ if limit == 0 || !limit.is_a?(Integer)
+ raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}")
+ end
+
gitaly_migrate(:find_commits) do |is_enabled|
if is_enabled
gitaly_commit_client.find_commits(options)
@@ -543,27 +546,34 @@ module Gitlab
end
# Returns the SHA of the most recent common ancestor of +from+ and +to+
- def merge_base_commit(from, to)
+ def merge_base(from, to)
gitaly_migrate(:merge_base) do |is_enabled|
if is_enabled
gitaly_repository_client.find_merge_base(from, to)
else
- rugged.merge_base(from, to)
+ rugged_merge_base(from, to)
end
end
end
- alias_method :merge_base, :merge_base_commit
# Gitaly note: JV: check gitlab-ee before removing this method.
def rugged_is_ancestor?(ancestor_id, descendant_id)
return false if ancestor_id.nil? || descendant_id.nil?
- merge_base_commit(ancestor_id, descendant_id) == ancestor_id
+ rugged_merge_base(ancestor_id, descendant_id) == ancestor_id
+ rescue Rugged::OdbError
+ false
end
# Returns true is +from+ is direct ancestor to +to+, otherwise false
def ancestor?(from, to)
- gitaly_commit_client.ancestor?(from, to)
+ Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled|
+ if is_enabled
+ gitaly_commit_client.ancestor?(from, to)
+ else
+ rugged_is_ancestor?(from, to)
+ end
+ end
end
def merged_branch_names(branch_names = [])
@@ -608,11 +618,11 @@ module Gitlab
if is_enabled
gitaly_ref_client.find_ref_name(sha, ref_path)
else
- args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha})
+ args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha})
# Not found -> ["", 0]
# Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0]
- popen(args, @path).first.split.last
+ run_git(args).first.split.last
end
end
end
@@ -670,11 +680,7 @@ module Gitlab
if is_enabled
gitaly_commit_client.commit_count(ref)
else
- walker = Rugged::Walker.new(rugged)
- walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
- oid = rugged.rev_parse_oid(ref)
- walker.push(oid)
- walker.count
+ rugged_commit_count(ref)
end
end
end
@@ -877,17 +883,12 @@ module Gitlab
end
def delete_refs(*ref_names)
- instructions = ref_names.map do |ref|
- "delete #{ref}\x00\x00"
- end
-
- command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z]
- message, status = popen(command, path) do |stdin|
- stdin.write(instructions.join)
- end
-
- unless status.zero?
- raise GitError.new("Could not delete refs #{ref_names}: #{message}")
+ gitaly_migrate(:delete_refs) do |is_enabled|
+ if is_enabled
+ gitaly_delete_refs(*ref_names)
+ else
+ git_delete_refs(*ref_names)
+ end
end
end
@@ -991,6 +992,18 @@ module Gitlab
attributes(path)[name]
end
+ # Check .gitattributes for a given ref
+ #
+ # This only checks the root .gitattributes file,
+ # it does not traverse subfolders to find additional .gitattributes files
+ #
+ # This method is around 30 times slower than `attributes`,
+ # which uses `$GIT_DIR/info/attributes`
+ def attributes_at(ref, file_path)
+ parser = AttributesAtRefParser.new(self, ref)
+ parser.attributes(file_path)
+ end
+
def languages(ref = nil)
Gitlab::GitalyClient.migrate(:commit_languages) do |is_enabled|
if is_enabled
@@ -1084,24 +1097,15 @@ module Gitlab
end
def write_ref(ref_path, ref, old_ref: nil, shell: true)
- if shell
- shell_write_ref(ref_path, ref, old_ref)
- else
- rugged_write_ref(ref_path, ref)
- end
- end
-
- def shell_write_ref(ref_path, ref, old_ref)
- raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
- raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
- raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00")
+ ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD"
- input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00"
- run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) }
- end
-
- def rugged_write_ref(ref_path, ref)
- rugged.references.create(ref_path, ref, force: true)
+ gitaly_migrate(:write_ref) do |is_enabled|
+ if is_enabled
+ gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell)
+ else
+ local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell)
+ end
+ end
end
def fetch_ref(source_repository, source_ref:, target_ref:)
@@ -1123,30 +1127,6 @@ module Gitlab
end
# Refactoring aid; allows us to copy code from app/models/repository.rb
- def run_git(args, chdir: path, env: {}, nice: false, &block)
- cmd = [Gitlab.config.git.bin_path, *args]
- cmd.unshift("nice") if nice
- circuit_breaker.perform do
- popen(cmd, chdir, env, &block)
- end
- end
-
- def run_git!(args, chdir: path, env: {}, nice: false, &block)
- output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block)
-
- raise GitError, output unless status.zero?
-
- output
- end
-
- # Refactoring aid; allows us to copy code from app/models/repository.rb
- def run_git_with_timeout(args, timeout, env: {})
- circuit_breaker.perform do
- popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env)
- end
- end
-
- # Refactoring aid; allows us to copy code from app/models/repository.rb
def commit(ref = 'HEAD')
Gitlab::Git::Commit.find(self, ref)
end
@@ -1200,6 +1180,19 @@ module Gitlab
end
end
+ def create_from_bundle(bundle_path)
+ gitaly_migrate(:create_repo_from_bundle) do |is_enabled|
+ if is_enabled
+ gitaly_repository_client.create_from_bundle(bundle_path)
+ else
+ run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil)
+ self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
+ end
+ end
+
+ true
+ end
+
def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:)
gitaly_migrate(:rebase) do |is_enabled|
if is_enabled
@@ -1229,33 +1222,13 @@ module Gitlab
end
def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:)
- squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)
- env = git_env_for_user(user).merge(
- 'GIT_AUTHOR_NAME' => author.name,
- 'GIT_AUTHOR_EMAIL' => author.email
- )
- diff_range = "#{start_sha}...#{end_sha}"
- diff_files = run_git!(
- %W(diff --name-only --diff-filter=a --binary #{diff_range})
- ).chomp
-
- with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
- # Apply diff of the `diff_range` to the worktree
- diff = run_git!(%W(diff --binary #{diff_range}))
- run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin|
- stdin.write(diff)
+ gitaly_migrate(:squash) do |is_enabled|
+ if is_enabled
+ gitaly_operation_client.user_squash(user, squash_id, branch,
+ start_sha, end_sha, author, message)
+ else
+ git_squash(user, squash_id, branch, start_sha, end_sha, author, message)
end
-
- # Commit the `diff_range` diff
- run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env)
-
- # Return the squash sha. May print a warning for ambiguous refs, but
- # we can ignore that with `--quiet` and just take the SHA, if present.
- # HEAD here always refers to the current HEAD commit, even if there is
- # another ref called HEAD.
- run_git!(
- %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env
- ).chomp
end
end
@@ -1281,44 +1254,47 @@ module Gitlab
success || gitlab_projects_error
end
+ def bundle_to_disk(save_path)
+ gitaly_migrate(:bundle_to_disk) do |is_enabled|
+ if is_enabled
+ gitaly_repository_client.create_bundle(save_path)
+ else
+ run_git!(%W(bundle create #{save_path} --all))
+ end
+ end
+
+ true
+ end
+
# rubocop:disable Metrics/ParameterLists
def multi_action(
user, branch_name:, message:, actions:,
author_email: nil, author_name: nil,
start_branch_name: nil, start_repository: self)
- OperationService.new(user, self).with_branch(
- branch_name,
- start_branch_name: start_branch_name,
- start_repository: start_repository
- ) do |start_commit|
- index = Gitlab::Git::Index.new(self)
- parents = []
-
- if start_commit
- index.read_tree(start_commit.rugged_commit.tree)
- parents = [start_commit.sha]
+ gitaly_migrate(:operation_user_commit_files) do |is_enabled|
+ if is_enabled
+ gitaly_operation_client.user_commit_files(user, branch_name,
+ message, actions, author_email, author_name,
+ start_branch_name, start_repository)
+ else
+ rugged_multi_action(user, branch_name, message, actions,
+ author_email, author_name, start_branch_name, start_repository)
end
-
- actions.each { |opts| index.apply(opts.delete(:action), opts) }
-
- committer = user_to_committer(user)
- author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer
- options = {
- tree: index.write_tree,
- message: message,
- parents: parents,
- author: author,
- committer: committer
- }
-
- create_commit(options)
end
end
# rubocop:enable Metrics/ParameterLists
def write_config(full_path:)
- rugged.config['gitlab.fullpath'] = full_path if full_path.present?
+ return unless full_path.present?
+
+ gitaly_migrate(:write_config) do |is_enabled|
+ if is_enabled
+ gitaly_repository_client.write_config(full_path: full_path)
+ else
+ rugged_write_config(full_path: full_path)
+ end
+ end
end
def gitaly_repository
@@ -1349,6 +1325,10 @@ module Gitlab
@gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self)
end
+ def gitaly_blob_client
+ @gitaly_blob_client ||= Gitlab::GitalyClient::BlobService.new(self)
+ end
+
def gitaly_conflicts_client(our_commit_oid, their_commit_oid)
Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid)
end
@@ -1363,8 +1343,166 @@ module Gitlab
raise CommandError.new(e)
end
+ def branch_names_contains_sha(sha)
+ gitaly_migrate(:branch_names_contains_sha) do |is_enabled|
+ if is_enabled
+ gitaly_ref_client.branch_names_contains_sha(sha)
+ else
+ refs_contains_sha(:branch, sha)
+ end
+ end
+ end
+
+ def tag_names_contains_sha(sha)
+ gitaly_migrate(:tag_names_contains_sha) do |is_enabled|
+ if is_enabled
+ gitaly_ref_client.tag_names_contains_sha(sha)
+ else
+ refs_contains_sha(:tag, sha)
+ end
+ end
+ end
+
+ def search_files_by_content(query, ref)
+ return [] if empty? || query.blank?
+
+ offset = 2
+ args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
+
+ run_git(args).first.scrub.split(/^--$/)
+ end
+
+ def can_be_merged?(source_sha, target_branch)
+ gitaly_migrate(:can_be_merged) do |is_enabled|
+ if is_enabled
+ gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target)
+ else
+ rugged_can_be_merged?(source_sha, target_branch)
+ end
+ end
+ end
+
+ def search_files_by_name(query, ref)
+ safe_query = Regexp.escape(query.sub(%r{^/*}, ""))
+
+ return [] if empty? || safe_query.blank?
+
+ args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{safe_query})
+
+ run_git(args).first.lines.map(&:strip)
+ end
+
+ def find_commits_by_message(query, ref, path, limit, offset)
+ gitaly_migrate(:commits_by_message) do |is_enabled|
+ if is_enabled
+ find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
+ else
+ find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
+ end
+ end
+ end
+
+ def shell_blame(sha, path)
+ output, _status = run_git(%W(blame -p #{sha} -- #{path}))
+ output
+ end
+
+ def can_be_merged?(source_sha, target_branch)
+ gitaly_migrate(:can_be_merged) do |is_enabled|
+ if is_enabled
+ gitaly_can_be_merged?(source_sha, find_branch(target_branch).target)
+ else
+ rugged_can_be_merged?(source_sha, target_branch)
+ end
+ end
+ end
+
+ def last_commit_id_for_path(sha, path)
+ gitaly_migrate(:last_commit_for_path) do |is_enabled|
+ if is_enabled
+ last_commit_for_path_by_gitaly(sha, path).id
+ else
+ last_commit_id_for_path_by_shelling_out(sha, path)
+ end
+ end
+ end
+
private
+ def local_write_ref(ref_path, ref, old_ref: nil, shell: true)
+ if shell
+ shell_write_ref(ref_path, ref, old_ref)
+ else
+ rugged_write_ref(ref_path, ref)
+ end
+ end
+
+ def refs_contains_sha(ref_type, sha)
+ args = %W(#{ref_type} --contains #{sha})
+ names = run_git(args).first
+
+ return [] unless names.respond_to?(:split)
+
+ names = names.split("\n").map(&:strip)
+
+ names.each do |name|
+ name.slice! '* '
+ end
+
+ names
+ end
+
+ def rugged_write_config(full_path:)
+ rugged.config['gitlab.fullpath'] = full_path
+ end
+
+ def shell_write_ref(ref_path, ref, old_ref)
+ raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ')
+ raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00")
+ raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00")
+
+ input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00"
+ run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) }
+ end
+
+ def rugged_write_ref(ref_path, ref)
+ rugged.references.create(ref_path, ref, force: true)
+ rescue Rugged::ReferenceError => ex
+ Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
+ rescue Rugged::OSError => ex
+ raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/
+
+ Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}"
+ end
+
+ def run_git(args, chdir: path, env: {}, nice: false, &block)
+ cmd = [Gitlab.config.git.bin_path, *args]
+ cmd.unshift("nice") if nice
+
+ object_directories = alternate_object_directories
+ if object_directories.any?
+ env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR)
+ end
+
+ circuit_breaker.perform do
+ popen(cmd, chdir, env, &block)
+ end
+ end
+
+ def run_git!(args, chdir: path, env: {}, nice: false, &block)
+ output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block)
+
+ raise GitError, output unless status.zero?
+
+ output
+ end
+
+ def run_git_with_timeout(args, timeout, env: {})
+ circuit_breaker.perform do
+ popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env)
+ end
+ end
+
def fresh_worktree?(path)
File.exist?(path) && !clean_stuck_worktree(path)
end
@@ -1379,7 +1517,7 @@ module Gitlab
if sparse_checkout_files
# Create worktree without checking out
run_git!(base_args + ['--no-checkout', worktree_path], env: env)
- worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path)
+ worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp
configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
@@ -1512,6 +1650,9 @@ module Gitlab
end
end
+ # Gitaly note: JV: although #log_by_shell shells out to Git I think the
+ # complexity is such that we should migrate it as Ruby before trying to
+ # do it in Go.
def log_by_shell(sha, options)
limit = options[:limit].to_i
offset = options[:offset].to_i
@@ -1522,7 +1663,7 @@ module Gitlab
offset_in_ruby = use_follow_flag && options[:offset].present?
limit += offset if offset_in_ruby
- cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log]
+ cmd = %w[log]
cmd << "--max-count=#{limit}"
cmd << '--format=%H'
cmd << "--skip=#{offset}" unless offset_in_ruby
@@ -1538,7 +1679,7 @@ module Gitlab
cmd += Array(options[:path])
end
- raw_output = IO.popen(cmd) { |io| io.read }
+ raw_output, _status = run_git(cmd)
lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines
lines.map! { |c| Rugged::Commit.new(rugged, c.strip) }
@@ -1576,18 +1717,23 @@ module Gitlab
end
def alternate_object_directories
- relative_paths = Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
+ relative_paths = relative_object_directories
if relative_paths.any?
relative_paths.map { |d| File.join(path, d) }
else
- Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES)
- .flatten
- .compact
- .flat_map { |d| d.split(File::PATH_SEPARATOR) }
+ absolute_object_directories.flat_map { |d| d.split(File::PATH_SEPARATOR) }
end
end
+ def relative_object_directories
+ Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
+ end
+
+ def absolute_object_directories
+ Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).flatten.compact
+ end
+
# Get the content of a blob for a given commit. If the blob is a commit
# (for submodules) then return the blob's OID.
def blob_content(commit, blob_name)
@@ -1731,13 +1877,13 @@ module Gitlab
def count_commits_by_shelling_out(options)
cmd = count_commits_shelling_command(options)
- raw_output = IO.popen(cmd) { |io| io.read }
+ raw_output, _status = run_git(cmd)
process_count_commits_raw_output(raw_output, options)
end
def count_commits_shelling_command(options)
- cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list]
+ cmd = %w[rev-list]
cmd << "--after=#{options[:after].iso8601}" if options[:after]
cmd << "--before=#{options[:before].iso8601}" if options[:before]
cmd << "--max-count=#{options[:max_count]}" if options[:max_count]
@@ -1782,20 +1928,17 @@ module Gitlab
return []
end
- cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree)
- cmd += %w(-r)
- cmd += %w(--full-tree)
- cmd += %w(--full-name)
- cmd += %W(-- #{actual_ref})
+ cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref})
+ raw_output, _status = run_git(cmd)
- raw_output = IO.popen(cmd, &:read).split("\n").map do |f|
+ lines = raw_output.split("\n").map do |f|
stuff, path = f.split("\t")
_mode, type, _sha = stuff.split(" ")
path if type == "blob"
# Contain only blob type
end
- raw_output.compact
+ lines.compact
end
# Returns true if the given ref name exists
@@ -1869,7 +2012,7 @@ module Gitlab
target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target)
Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit)
rescue Rugged::ReferenceError => e
- raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/
+ raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'}
raise InvalidRef.new("Invalid reference #{start_point}")
end
@@ -2019,6 +2162,37 @@ module Gitlab
end
end
+ def git_squash(user, squash_id, branch, start_sha, end_sha, author, message)
+ squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)
+ env = git_env_for_user(user).merge(
+ 'GIT_AUTHOR_NAME' => author.name,
+ 'GIT_AUTHOR_EMAIL' => author.email
+ )
+ diff_range = "#{start_sha}...#{end_sha}"
+ diff_files = run_git!(
+ %W(diff --name-only --diff-filter=a --binary #{diff_range})
+ ).chomp
+
+ with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do
+ # Apply diff of the `diff_range` to the worktree
+ diff = run_git!(%W(diff --binary #{diff_range}))
+ run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin|
+ stdin.write(diff)
+ end
+
+ # Commit the `diff_range` diff
+ run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env)
+
+ # Return the squash sha. May print a warning for ambiguous refs, but
+ # we can ignore that with `--quiet` and just take the SHA, if present.
+ # HEAD here always refers to the current HEAD commit, even if there is
+ # another ref called HEAD.
+ run_git!(
+ %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env
+ ).chomp
+ end
+ end
+
def local_fetch_ref(source_path, source_ref:, target_ref:)
args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
run_git(args)
@@ -2042,7 +2216,7 @@ module Gitlab
source_sha
end
- rescue Rugged::ReferenceError
+ rescue Rugged::ReferenceError, InvalidRef
raise ArgumentError, 'Invalid merge source'
end
@@ -2054,6 +2228,24 @@ module Gitlab
remote_update(remote_name, url: url)
end
+ def git_delete_refs(*ref_names)
+ instructions = ref_names.map do |ref|
+ "delete #{ref}\x00\x00"
+ end
+
+ message, status = run_git(%w[update-ref --stdin -z]) do |stdin|
+ stdin.write(instructions.join)
+ end
+
+ unless status.zero?
+ raise GitError.new("Could not delete refs #{ref_names}: #{message}")
+ end
+ end
+
+ def gitaly_delete_refs(*ref_names)
+ gitaly_ref_client.delete_refs(refs: ref_names)
+ end
+
def rugged_remove_remote(remote_name)
# When a remote is deleted all its remote refs are deleted too, but in
# the case of mirrors we map its refs (that would usualy go under
@@ -2079,13 +2271,107 @@ module Gitlab
remove_remote(remote_name)
end
+ def rugged_multi_action(
+ user, branch_name, message, actions, author_email, author_name,
+ start_branch_name, start_repository)
+
+ OperationService.new(user, self).with_branch(
+ branch_name,
+ start_branch_name: start_branch_name,
+ start_repository: start_repository
+ ) do |start_commit|
+ index = Gitlab::Git::Index.new(self)
+ parents = []
+
+ if start_commit
+ index.read_tree(start_commit.rugged_commit.tree)
+ parents = [start_commit.sha]
+ end
+
+ actions.each { |opts| index.apply(opts.delete(:action), opts) }
+
+ committer = user_to_committer(user)
+ author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer
+ options = {
+ tree: index.write_tree,
+ message: message,
+ parents: parents,
+ author: author,
+ committer: committer
+ }
+
+ create_commit(options)
+ end
+ end
+
def fetch_remote(remote_name = 'origin', env: nil)
run_git(['fetch', remote_name], env: env).last.zero?
end
+ def gitaly_can_be_merged?(their_commit, our_commit)
+ !gitaly_conflicts_client(our_commit, their_commit).conflicts?
+ end
+
+ def rugged_can_be_merged?(their_commit, our_commit)
+ !rugged.merge_commits(our_commit, their_commit).conflicts?
+ end
+
def gitlab_projects_error
raise CommandError, @gitlab_projects.output
end
+
+ def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
+ ref ||= root_ref
+
+ args = %W(
+ log #{ref} --pretty=%H --skip #{offset}
+ --max-count #{limit} --grep=#{query} --regexp-ignore-case
+ )
+ args = args.concat(%W(-- #{path})) if path.present?
+
+ git_log_results = run_git(args).first.lines
+
+ git_log_results.map { |c| commit(c.chomp) }.compact
+ end
+
+ def find_commits_by_message_by_gitaly(query, ref, path, limit, offset)
+ gitaly_commit_client
+ .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset)
+ .map { |c| commit(c) }
+ end
+
+ def gitaly_can_be_merged?(their_commit, our_commit)
+ !gitaly_conflicts_client(our_commit, their_commit).conflicts?
+ end
+
+ def rugged_can_be_merged?(their_commit, our_commit)
+ !rugged.merge_commits(our_commit, their_commit).conflicts?
+ end
+
+ def last_commit_for_path_by_gitaly(sha, path)
+ gitaly_commit_client.last_commit_for_path(sha, path)
+ end
+
+ def last_commit_id_for_path_by_shelling_out(sha, path)
+ args = %W(rev-list --max-count=1 #{sha} -- #{path})
+ run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip
+ end
+
+ def rugged_merge_base(from, to)
+ rugged.merge_base(from, to)
+ rescue Rugged::ReferenceError
+ nil
+ end
+
+ def rugged_commit_count(ref)
+ walker = Rugged::Walker.new(rugged)
+ walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE)
+ oid = rugged.rev_parse_oid(ref)
+ walker.push(oid)
+ walker.count
+ rescue Rugged::ReferenceError
+ 0
+ end
end
end
end
diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb
index effb1f0ca19..dc424a433fb 100644
--- a/lib/gitlab/git/repository_mirroring.rb
+++ b/lib/gitlab/git/repository_mirroring.rb
@@ -43,7 +43,7 @@ module Gitlab
branches = []
rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
- name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+ name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '')
begin
target_commit = Gitlab::Git::Commit.find(self, ref.target)
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index bc4e160dce9..8a8f7b051ed 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -1,5 +1,3 @@
-# Gitaly note: JV: no RPC's here.
-#
module Gitlab
module Git
class Tag < Ref
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index 5cf336af3c6..ba6058fd3c9 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -83,6 +83,8 @@ module Gitlab
commit_id: sha
)
end
+ rescue Rugged::ReferenceError
+ []
end
end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 4ed78daa443..39040d56971 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -25,8 +25,9 @@ module Gitlab
@repository.exists?
end
+ # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539
def write_page(name, format, content, commit_details)
- @repository.gitaly_migrate(:wiki_write_page) do |is_enabled|
+ @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
if is_enabled
gitaly_write_page(name, format, content, commit_details)
gollum_wiki.clear_cache
@@ -47,8 +48,9 @@ module Gitlab
end
end
+ # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094
def update_page(page_path, title, format, content, commit_details)
- @repository.gitaly_migrate(:wiki_update_page) do |is_enabled|
+ @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
if is_enabled
gitaly_update_page(page_path, title, format, content, commit_details)
gollum_wiki.clear_cache
@@ -68,8 +70,9 @@ module Gitlab
end
end
+ # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039
def page(title:, version: nil, dir: nil)
- @repository.gitaly_migrate(:wiki_find_page) do |is_enabled|
+ @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled|
if is_enabled
gitaly_find_page(title: title, version: version, dir: dir)
else
@@ -129,6 +132,20 @@ module Gitlab
page.url_path
end
+ def page_formatted_data(title:, dir: nil, version: nil)
+ version = version&.id
+
+ @repository.gitaly_migrate(:wiki_page_formatted_data) do |is_enabled|
+ if is_enabled
+ gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version)
+ else
+ # We don't use #page because if wiki_find_page feature is enabled, we would
+ # get a page without formatted_data.
+ gollum_find_page(title: title, dir: dir, version: version)&.formatted_data
+ end
+ end
+ end
+
private
# options:
@@ -190,7 +207,10 @@ module Gitlab
assert_type!(format, Symbol)
assert_type!(commit_details, CommitDetails)
- gollum_wiki.write_page(name, format, content, commit_details.to_h)
+ filename = File.basename(name)
+ dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir
+
+ gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir)
nil
rescue Gollum::DuplicatePageError => e
@@ -208,7 +228,15 @@ module Gitlab
assert_type!(format, Symbol)
assert_type!(commit_details, CommitDetails)
- gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h)
+ page = gollum_page_by_path(page_path)
+ committer = Gollum::Committer.new(page.wiki, commit_details.to_h)
+
+ # Instead of performing two renames if the title has changed,
+ # the update_page will only update the format and content and
+ # the rename_page will do anything related to moving/renaming
+ gollum_wiki.update_page(page, page.name, format, content, committer: committer)
+ gollum_wiki.rename_page(page, title, committer: committer)
+ committer.commit
nil
end
diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb
index a06bac4414f..669ae11a423 100644
--- a/lib/gitlab/git/wiki_page.rb
+++ b/lib/gitlab/git/wiki_page.rb
@@ -1,7 +1,7 @@
module Gitlab
module Git
class WikiPage
- attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical
+ attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data
# This class is meant to be serializable so that it can be constructed
# by Gitaly and sent over the network to GitLab.
@@ -21,6 +21,7 @@ module Gitlab
@raw_data = gollum_page.raw_data
@name = gollum_page.name
@historical = gollum_page.historical?
+ @formatted_data = gollum_page.formatted_data if gollum_page.is_a?(Gollum::Page)
@version = version
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 4507ea923b4..c5d3e944f7d 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -1,9 +1,12 @@
require 'base64'
require 'gitaly'
+require 'grpc/health/v1/health_pb'
+require 'grpc/health/v1/health_services_pb'
module Gitlab
module GitalyClient
+ include Gitlab::Metrics::Methods
module MigrationStatus
DISABLED = 1
OPT_IN = 2
@@ -31,8 +34,6 @@ module Gitlab
CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
MUTEX = Mutex.new
- METRICS_MUTEX = Mutex.new
- private_constant :MUTEX, :METRICS_MUTEX
class << self
attr_accessor :query_time
@@ -40,28 +41,14 @@ module Gitlab
self.query_time = 0
- def self.migrate_histogram
- @migrate_histogram ||=
- METRICS_MUTEX.synchronize do
- # If a thread was blocked on the mutex, the value was set already
- return @migrate_histogram if @migrate_histogram
-
- Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds,
- "Gitaly migration call execution timings",
- gitaly_enabled: nil, feature: nil)
- end
+ define_histogram :gitaly_migrate_call_duration_seconds do
+ docstring "Gitaly migration call execution timings"
+ base_labels gitaly_enabled: nil, feature: nil
end
- def self.gitaly_call_histogram
- @gitaly_call_histogram ||=
- METRICS_MUTEX.synchronize do
- # If a thread was blocked on the mutex, the value was set already
- return @gitaly_call_histogram if @gitaly_call_histogram
-
- Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds,
- "Gitaly endpoint histogram by controller and action combination",
- Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil))
- end
+ define_histogram :gitaly_controller_action_duration_seconds do
+ docstring "Gitaly endpoint histogram by controller and action combination"
+ base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)
end
def self.stub(name, storage)
@@ -69,14 +56,27 @@ module Gitlab
@stubs ||= {}
@stubs[storage] ||= {}
@stubs[storage][name] ||= begin
- klass = Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
- addr = address(storage)
- addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+ klass = stub_class(name)
+ addr = stub_address(storage)
klass.new(addr, :this_channel_is_insecure)
end
end
end
+ def self.stub_class(name)
+ if name == :health_check
+ Grpc::Health::V1::Health::Stub
+ else
+ Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
+ end
+ end
+
+ def self.stub_address(storage)
+ addr = address(storage)
+ addr = addr.sub(%r{^tcp://}, '') if URI(addr).scheme == 'tcp'
+ addr
+ end
+
def self.clear_stubs!
MUTEX.synchronize do
@stubs = nil
@@ -130,7 +130,7 @@ module Gitlab
# Keep track, seperately, for the performance bar
self.query_time += duration
- gitaly_call_histogram.observe(
+ gitaly_controller_action_duration_seconds.observe(
current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s),
duration)
end
@@ -232,7 +232,7 @@ module Gitlab
yield is_enabled
ensure
total_time = Gitlab::Metrics::System.monotonic_time - start
- migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time)
+ gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time)
feature_stack.shift
Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty?
end
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index a250eb75bd4..d70a1a7665e 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -32,6 +32,28 @@ module Gitlab
binary: Gitlab::Git::Blob.binary?(data)
)
end
+
+ def batch_lfs_pointers(blob_ids)
+ return [] if blob_ids.empty?
+
+ request = Gitaly::GetLFSPointersRequest.new(
+ repository: @gitaly_repo,
+ blob_ids: blob_ids
+ )
+
+ response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request)
+
+ response.flat_map do |message|
+ message.lfs_pointers.map do |lfs_pointer|
+ Gitlab::Git::Blob.new(
+ id: lfs_pointer.oid,
+ size: lfs_pointer.size,
+ data: lfs_pointer.data,
+ binary: Gitlab::Git::Blob.binary?(lfs_pointer.data)
+ )
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 2231371cfa7..5767f06b0ce 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -38,19 +38,27 @@ module Gitlab
from_id = case from
when NilClass
EMPTY_TREE_ID
- when Rugged::Commit
- from.oid
else
- from
+ if from.respond_to?(:oid)
+ # This is meant to match a Rugged::Commit. This should be impossible in
+ # the future.
+ from.oid
+ else
+ from
+ end
end
to_id = case to
when NilClass
EMPTY_TREE_ID
- when Rugged::Commit
- to.oid
else
- to
+ if to.respond_to?(:oid)
+ # This is meant to match a Rugged::Commit. This should be impossible in
+ # the future.
+ to.oid
+ else
+ to
+ end
end
request_params = diff_between_commits_request_params(from_id, to_id, options)
@@ -125,11 +133,11 @@ module Gitlab
def commit_count(ref, options = {})
request = Gitaly::CountCommitsRequest.new(
repository: @gitaly_repo,
- revision: ref
+ revision: encode_binary(ref)
)
request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
- request.path = options[:path] if options[:path].present?
+ request.path = encode_binary(options[:path]) if options[:path].present?
request.max_count = options[:max_count] if options[:max_count].present?
GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count
@@ -249,7 +257,7 @@ module Gitlab
offset: options[:offset],
follow: options[:follow],
skip_merges: options[:skip_merges],
- disable_walk: options[:disable_walk]
+ disable_walk: true # This option is deprecated. The 'walk' implementation is being removed.
)
request.after = GitalyClient.timestamp(options[:after]) if options[:after]
request.before = GitalyClient.timestamp(options[:before]) if options[:before]
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index 2565d537aff..e14734495a8 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -25,6 +25,11 @@ module Gitlab
def conflicts?
list_conflict_files.any?
+ rescue GRPC::FailedPrecondition
+ # The server raises this exception when it encounters ConflictSideMissing, which
+ # means a conflict exists but its `theirs` or `ours` data is nil due to a non-existent
+ # file in one of the trees.
+ true
end
def resolve_conflicts(target_repository, resolution, source_branch, target_branch)
diff --git a/lib/gitlab/gitaly_client/health_check_service.rb b/lib/gitlab/gitaly_client/health_check_service.rb
new file mode 100644
index 00000000000..6c1213f5e20
--- /dev/null
+++ b/lib/gitlab/gitaly_client/health_check_service.rb
@@ -0,0 +1,19 @@
+module Gitlab
+ module GitalyClient
+ class HealthCheckService
+ def initialize(storage)
+ @storage = storage
+ end
+
+ # Sends a gRPC health ping to the Gitaly server for the storage shard.
+ def check
+ request = Grpc::Health::V1::HealthCheckRequest.new
+ response = GitalyClient.call(@storage, :health_check, :check, request, timeout: GitalyClient.fast_timeout)
+
+ { success: response&.status == :SERVING }
+ rescue GRPC::BadStatus => e
+ { success: false, message: e.to_s }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 7319de69d13..831cfd1e014 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -3,6 +3,8 @@ module Gitlab
class OperationService
include Gitlab::EncodingHelper
+ MAX_MSG_SIZE = 128.kilobytes.freeze
+
def initialize(repository)
@gitaly_repo = repository.gitaly_repository
@repository = repository
@@ -101,7 +103,13 @@ module Gitlab
request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true))
- branch_update = response_enum.next.branch_update
+ second_response = response_enum.next
+
+ if second_response.pre_receive_error.present?
+ raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error
+ end
+
+ branch_update = second_response.branch_update
return if branch_update.nil?
raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present?
@@ -175,6 +183,75 @@ module Gitlab
end
end
+ def user_squash(user, squash_id, branch, start_sha, end_sha, author, message)
+ request = Gitaly::UserSquashRequest.new(
+ repository: @gitaly_repo,
+ user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+ squash_id: squash_id.to_s,
+ branch: encode_binary(branch),
+ start_sha: start_sha,
+ end_sha: end_sha,
+ author: Gitlab::Git::User.from_gitlab(author).to_gitaly,
+ commit_message: encode_binary(message)
+ )
+
+ response = GitalyClient.call(
+ @repository.storage,
+ :operation_service,
+ :user_squash,
+ request
+ )
+
+ if response.git_error.presence
+ raise Gitlab::Git::Repository::GitError, response.git_error
+ end
+
+ response.squash_sha
+ end
+
+ def user_commit_files(
+ user, branch_name, commit_message, actions, author_email, author_name,
+ start_branch_name, start_repository)
+
+ req_enum = Enumerator.new do |y|
+ header = user_commit_files_request_header(user, branch_name,
+ commit_message, actions, author_email, author_name,
+ start_branch_name, start_repository)
+
+ y.yield Gitaly::UserCommitFilesRequest.new(header: header)
+
+ actions.each do |action|
+ action_header = user_commit_files_action_header(action)
+ y.yield Gitaly::UserCommitFilesRequest.new(
+ action: Gitaly::UserCommitFilesAction.new(header: action_header)
+ )
+
+ reader = binary_stringio(action[:content])
+
+ until reader.eof?
+ chunk = reader.read(MAX_MSG_SIZE)
+
+ y.yield Gitaly::UserCommitFilesRequest.new(
+ action: Gitaly::UserCommitFilesAction.new(content: chunk)
+ )
+ end
+ end
+ end
+
+ response = GitalyClient.call(@repository.storage, :operation_service,
+ :user_commit_files, req_enum, remote_storage: start_repository.storage)
+
+ if (pre_receive_error = response.pre_receive_error.presence)
+ raise Gitlab::Git::HooksService::PreReceiveError, pre_receive_error
+ end
+
+ if (index_error = response.index_error.presence)
+ raise Gitlab::Git::Index::IndexError, index_error
+ end
+
+ Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
+ end
+
private
def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
@@ -212,6 +289,33 @@ module Gitlab
Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
end
end
+
+ def user_commit_files_request_header(
+ user, branch_name, commit_message, actions, author_email, author_name,
+ start_branch_name, start_repository)
+
+ Gitaly::UserCommitFilesRequestHeader.new(
+ repository: @gitaly_repo,
+ user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
+ branch_name: encode_binary(branch_name),
+ commit_message: encode_binary(commit_message),
+ commit_author_name: encode_binary(author_name),
+ commit_author_email: encode_binary(author_email),
+ start_branch_name: encode_binary(start_branch_name),
+ start_repository: start_repository.gitaly_repository
+ )
+ end
+
+ def user_commit_files_action_header(action)
+ Gitaly::UserCommitFilesActionHeader.new(
+ action: action[:action].upcase.to_sym,
+ file_path: encode_binary(action[:file_path]),
+ previous_path: encode_binary(action[:previous_path]),
+ base64_content: action[:encoding] == 'base64'
+ )
+ rescue RangeError
+ raise ArgumentError, "Unknown action '#{action[:action]}'"
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index f8e2a27f3fe..ba6b577fd17 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -133,13 +133,42 @@ module Gitlab
GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
end
- def delete_refs(except_with_prefixes:)
+ def delete_refs(refs: [], except_with_prefixes: [])
request = Gitaly::DeleteRefsRequest.new(
repository: @gitaly_repo,
- except_with_prefix: except_with_prefixes
+ refs: refs.map { |r| encode_binary(r) },
+ except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) }
)
- GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request)
+ response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request)
+
+ raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present?
+ end
+
+ # Limit: 0 implies no limit, thus all tag names will be returned
+ def tag_names_contains_sha(sha, limit: 0)
+ request = Gitaly::ListTagNamesContainingCommitRequest.new(
+ repository: @gitaly_repo,
+ commit_id: sha,
+ limit: limit
+ )
+
+ stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request)
+
+ consume_ref_contains_sha_response(stream, :tag_names)
+ end
+
+ # Limit: 0 implies no limit, thus all tag names will be returned
+ def branch_names_contains_sha(sha, limit: 0)
+ request = Gitaly::ListBranchNamesContainingCommitRequest.new(
+ repository: @gitaly_repo,
+ commit_id: sha,
+ limit: limit
+ )
+
+ stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request)
+
+ consume_ref_contains_sha_response(stream, :branch_names)
end
private
@@ -212,6 +241,13 @@ module Gitlab
Gitlab::Git::Commit.decorate(@repository, hash)
end
+ def consume_ref_contains_sha_response(stream, collection_name)
+ stream.each_with_object([]) do |response, array|
+ encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend
+ array.concat(encoded_names)
+ end
+ end
+
def invalid_ref!(message)
raise Gitlab::Git::Repository::InvalidRef.new(message)
end
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index e58f641d69a..58c356edfd1 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -1,6 +1,8 @@
module Gitlab
module GitalyClient
class RemoteService
+ MAX_MSG_SIZE = 128.kilobytes.freeze
+
def initialize(repository)
@repository = repository
@gitaly_repo = repository.gitaly_repository
@@ -38,6 +40,31 @@ module Gitlab
response.result
end
+
+ def update_remote_mirror(ref_name, only_branches_matching)
+ req_enum = Enumerator.new do |y|
+ y.yield Gitaly::UpdateRemoteMirrorRequest.new(
+ repository: @gitaly_repo,
+ ref_name: ref_name
+ )
+
+ current_size = 0
+
+ slices = only_branches_matching.slice_before do |branch_name|
+ current_size += branch_name.bytesize
+
+ next false if current_size < MAX_MSG_SIZE
+
+ current_size = 0
+ end
+
+ slices.each do |slice|
+ y.yield Gitaly::UpdateRemoteMirrorRequest.new(only_branches_matching: slice)
+ end
+ end
+
+ GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum)
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 12016aee2a6..60706b4f0d8 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -3,6 +3,8 @@ module Gitlab
class RepositoryService
include Gitlab::EncodingHelper
+ MAX_MSG_SIZE = 128.kilobytes.freeze
+
def initialize(repository)
@repository = repository
@gitaly_repo = repository.gitaly_repository
@@ -161,6 +163,75 @@ module Gitlab
return response.error.b, 1
end
end
+
+ def create_bundle(save_path)
+ request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
+ response = GitalyClient.call(
+ @storage,
+ :repository_service,
+ :create_bundle,
+ request,
+ timeout: GitalyClient.default_timeout
+ )
+
+ File.open(save_path, 'wb') do |f|
+ response.each do |message|
+ f.write(message.data)
+ end
+ end
+ end
+
+ def create_from_bundle(bundle_path)
+ request = Gitaly::CreateRepositoryFromBundleRequest.new(repository: @gitaly_repo)
+ enum = Enumerator.new do |y|
+ File.open(bundle_path, 'rb') do |f|
+ while data = f.read(MAX_MSG_SIZE)
+ request.data = data
+
+ y.yield request
+
+ request = Gitaly::CreateRepositoryFromBundleRequest.new
+ end
+ end
+ end
+
+ GitalyClient.call(
+ @storage,
+ :repository_service,
+ :create_repository_from_bundle,
+ enum,
+ timeout: GitalyClient.default_timeout
+ )
+ end
+
+ def write_ref(ref_path, ref, old_ref, shell)
+ request = Gitaly::WriteRefRequest.new(
+ repository: @gitaly_repo,
+ ref: ref_path.b,
+ revision: ref.b,
+ shell: shell
+ )
+ request.old_revision = old_ref.b unless old_ref.nil?
+
+ response = GitalyClient.call(@storage, :repository_service, :write_ref, request)
+
+ raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present?
+
+ true
+ end
+
+ def write_config(full_path:)
+ request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path)
+ response = GitalyClient.call(
+ @storage,
+ :repository_service,
+ :write_config,
+ request,
+ timeout: GitalyClient.fast_timeout
+ )
+
+ raise Gitlab::Git::OSError.new(response.error) unless response.error.empty?
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb
new file mode 100644
index 00000000000..2e1076d1f66
--- /dev/null
+++ b/lib/gitlab/gitaly_client/server_service.rb
@@ -0,0 +1,16 @@
+module Gitlab
+ module GitalyClient
+ # Meant for extraction of server data, and later maybe to perform misc task
+ #
+ # Not meant for connection logic, look in Gitlab::GitalyClient
+ class ServerService
+ def initialize(storage)
+ @storage = storage
+ end
+
+ def info
+ GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb
index 7339468e911..a02d15db5dd 100644
--- a/lib/gitlab/gitaly_client/wiki_page.rb
+++ b/lib/gitlab/gitaly_client/wiki_page.rb
@@ -4,6 +4,7 @@ module Gitlab
ATTRS = %i(title format url_path path name historical raw_data).freeze
include AttributesBag
+ include Gitlab::EncodingHelper
def initialize(params)
super
@@ -11,6 +12,10 @@ module Gitlab
# All gRPC strings in a response are frozen, so we get an unfrozen
# version here so appending to `raw_data` doesn't blow up.
@raw_data = @raw_data.dup
+
+ @title = encode_utf8(@title)
+ @path = encode_utf8(@path)
+ @name = encode_utf8(@name)
end
def historical?
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index a98c3c0b160..0d8dd5cb8f4 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -151,6 +151,18 @@ module Gitlab
wiki_file
end
+ def get_formatted_data(title:, dir: nil, version: nil)
+ request = Gitaly::WikiGetFormattedDataRequest.new(
+ repository: @gitaly_repo,
+ title: encode_binary(title),
+ revision: encode_binary(version),
+ directory: encode_binary(dir)
+ )
+
+ response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request)
+ response.reduce("") { |memo, msg| memo << msg.data }
+ end
+
private
# If a block is given and the yielded value is truthy, iteration will be
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index 5437e32e9f1..e70361c163b 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -57,10 +57,7 @@ module Gitlab
end
def commit_exists?(sha)
- project.repository.lookup(sha)
- true
- rescue Rugged::Error
- false
+ project.repository.commit(sha).present?
end
def collection_method
diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb
index bb7439a0641..be1334ca98a 100644
--- a/lib/gitlab/github_import/representation/diff_note.rb
+++ b/lib/gitlab/github_import/representation/diff_note.rb
@@ -13,7 +13,7 @@ module Gitlab
:diff_hunk, :author, :note, :created_at, :updated_at,
:github_id
- NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i
+ NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i
# Builds a diff note from a GitHub API response.
#
diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb
index a68bc4c002f..070e3b2db8d 100644
--- a/lib/gitlab/github_import/representation/note.rb
+++ b/lib/gitlab/github_import/representation/note.rb
@@ -12,7 +12,7 @@ module Gitlab
expose_attribute :noteable_id, :noteable_type, :author, :note,
:created_at, :updated_at, :github_id
- NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i
+ NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i
# Builds a note from a GitHub API response.
#
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 9148d7571f2..86a90d57d9c 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -3,12 +3,11 @@
module Gitlab
module GonHelper
include WebpackHelper
- include Gitlab::CurrentSettings
def add_gon_variables
gon.api_version = 'v4'
gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
- gon.max_file_size = current_application_settings.max_attachment_size
+ gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size
gon.asset_host = ActionController::Base.asset_host
gon.webpack_public_path = webpack_public_path
gon.relative_url_root = Gitlab.config.gitlab.relative_url_root
@@ -16,7 +15,7 @@ module Gitlab
gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class
gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css')
gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js')
- gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled
+ gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled
gon.gitlab_url = Gitlab.config.gitlab.url
gon.revision = Gitlab::REVISION
gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png')
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
new file mode 100644
index 00000000000..11416c002e3
--- /dev/null
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -0,0 +1,53 @@
+module Gitlab
+ module HealthChecks
+ class GitalyCheck
+ extend BaseAbstractCheck
+
+ METRIC_PREFIX = 'gitaly_health_check'.freeze
+
+ class << self
+ def readiness
+ repository_storages.map do |storage_name|
+ check(storage_name)
+ end
+ end
+
+ def metrics
+ repository_storages.flat_map do |storage_name|
+ result, elapsed = with_timing { check(storage_name) }
+ labels = { shard: storage_name }
+
+ [
+ metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels),
+ metric("#{metric_prefix}_latency_seconds", elapsed, **labels)
+ ].flatten
+ end
+ end
+
+ def check(storage_name)
+ serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name)
+ result = serv.check
+ HealthChecks::Result.new(result[:success], result[:message], shard: storage_name)
+ end
+
+ private
+
+ def metric_prefix
+ METRIC_PREFIX
+ end
+
+ def successful?(result)
+ result[:success]
+ end
+
+ def repository_storages
+ storages.keys
+ end
+
+ def storages
+ Gitlab.config.repositories.storages
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index dd5d35feab9..2f163db936b 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -11,15 +11,6 @@ module Gitlab
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
- def git_bundle(repo_path:, bundle_path:)
- execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
- end
-
- def git_clone_bundle(repo_path:, bundle_path:)
- execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path}))
- Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path))
- end
-
def mkdir_p(path)
FileUtils.mkdir_p(path, mode: DEFAULT_MODE)
FileUtils.chmod(DEFAULT_MODE, path)
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 5c971564a73..0f4c3498036 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -59,7 +59,7 @@ module Gitlab
end
def extracted_files
- Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ }
+ Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} }
end
end
end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index cb711a83433..759833a5ee5 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -139,13 +139,12 @@ module Gitlab
end
def setup_label
- return unless @relation_hash['type'] == 'GroupLabel'
-
# If there's no group, move the label to a project label
- if @relation_hash['group_id']
+ if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id']
@relation_hash['project_id'] = nil
@relation_name = :group_label
else
+ @relation_hash['group_id'] = nil
@relation_hash['type'] = 'ProjectLabel'
end
end
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index d0e5cfcfd3e..5a9bbceac67 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -13,7 +13,7 @@ module Gitlab
def restore
return true unless File.exist?(@path_to_bundle)
- git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle)
+ @project.repository.create_from_bundle(@path_to_bundle)
rescue => e
@shared.error(e)
false
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index a7028a32570..695462c7dd2 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -21,7 +21,7 @@ module Gitlab
def bundle_to_disk
mkdir_p(@shared.export_path)
- git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+ @project.repository.bundle_to_disk(@full_path)
rescue => e
@shared.error(e)
false
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index d03cbc880fd..b34cafc6876 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -19,8 +19,13 @@ module Gitlab
def error(error)
error_out(error.message, caller[0].dup)
@errors << error.message
+
# Debug:
- Rails.logger.error(error.backtrace.join("\n"))
+ if error.backtrace
+ Rails.logger.error("Import/Export backtrace: #{error.backtrace.join("\n")}")
+ else
+ Rails.logger.error("No backtrace found")
+ end
end
private
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index 627a487d577..2f08dda55fd 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -17,15 +17,13 @@ module Gitlab
false
end
- private
+ def uploads_path
+ FileUploader.absolute_base_dir(@project)
+ end
def uploads_export_path
File.join(@shared.export_path, 'uploads')
end
-
- def uploads_path
- FileUploader.dynamic_path_segment(@project)
- end
end
end
end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 1e6722a7bba..5fa2e101e29 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -10,7 +10,7 @@ module Gitlab
def bundle_to_disk(full_path)
mkdir_p(@shared.export_path)
- git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+ @wiki.repository.bundle_to_disk(full_path)
rescue => e
@shared.error(e)
false
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
index a3216759cae..ca5e06009fa 100644
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ b/lib/gitlab/kubernetes/helm/pod.rb
@@ -64,7 +64,7 @@ module Gitlab
{
name: 'configuration-volume',
configMap: {
- name: 'values-content-configuration',
+ name: "values-content-configuration-#{command.name}",
items: [{ key: 'values', path: 'values.yaml' }]
}
}
@@ -81,7 +81,11 @@ module Gitlab
def create_config_map
resource = ::Kubeclient::Resource.new
- resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } }
+ resource.metadata = {
+ name: "values-content-configuration-#{command.name}",
+ namespace: namespace_name,
+ labels: { name: "values-content-configuration-#{command.name}" }
+ }
resource.data = { values: File.read(command.chart_values_file) }
kubeclient.create_config_map(resource)
end
diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb
index 1bd0965679a..96171dc26c4 100644
--- a/lib/gitlab/ldap/auth_hash.rb
+++ b/lib/gitlab/ldap/auth_hash.rb
@@ -7,6 +7,12 @@ module Gitlab
@uid ||= Gitlab::LDAP::Person.normalize_dn(super)
end
+ def username
+ super.tap do |username|
+ username.downcase! if ldap_config.lowercase_usernames
+ end
+ end
+
private
def get_info(key)
diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb
index cde60addcf7..47b3fce3e7a 100644
--- a/lib/gitlab/ldap/config.rb
+++ b/lib/gitlab/ldap/config.rb
@@ -139,6 +139,10 @@ module Gitlab
options['allow_username_or_email_login']
end
+ def lowercase_usernames
+ options['lowercase_usernames']
+ end
+
def name_proc
if allow_username_or_email_login
proc { |name| name.gsub(/@.*\z/, '') }
diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb
index e81cec6ba1a..b91757c2a4b 100644
--- a/lib/gitlab/ldap/person.rb
+++ b/lib/gitlab/ldap/person.rb
@@ -82,7 +82,9 @@ module Gitlab
# be returned. We need only one for username.
# Ex. `uid` returns only one value but `mail` may
# return an array of multiple email addresses.
- [username].flatten.first
+ [username].flatten.first.tap do |username|
+ username.downcase! if config.lowercase_usernames
+ end
end
def email
diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb
index 41e7eac4d08..cbabe5454ca 100644
--- a/lib/gitlab/legacy_github_import/project_creator.rb
+++ b/lib/gitlab/legacy_github_import/project_creator.rb
@@ -1,8 +1,6 @@
module Gitlab
module LegacyGithubImport
class ProjectCreator
- include Gitlab::CurrentSettings
-
attr_reader :repo, :name, :namespace, :current_user, :session_data, :type
def initialize(repo, name, namespace, current_user, session_data, type: 'github')
@@ -36,7 +34,7 @@ module Gitlab
end
def visibility_level
- repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility
+ repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility
end
#
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 4779755bb22..7d63ca5627d 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,7 +1,7 @@
module Gitlab
module Metrics
- extend Gitlab::Metrics::InfluxDb
- extend Gitlab::Metrics::Prometheus
+ include Gitlab::Metrics::InfluxDb
+ include Gitlab::Metrics::Prometheus
def self.enabled?
influx_metrics_enabled? || prometheus_metrics_enabled?
diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb
index ef44a13df51..66f30e3b397 100644
--- a/lib/gitlab/metrics/influx_db.rb
+++ b/lib/gitlab/metrics/influx_db.rb
@@ -1,179 +1,187 @@
module Gitlab
module Metrics
module InfluxDb
- include Gitlab::CurrentSettings
- extend self
+ extend ActiveSupport::Concern
+ include Gitlab::Metrics::Methods
+
+ EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
MUTEX = Mutex.new
private_constant :MUTEX
- def influx_metrics_enabled?
- settings[:enabled] || false
- end
+ class_methods do
+ def influx_metrics_enabled?
+ settings[:enabled] || false
+ end
- # Prometheus histogram buckets used for arbitrary code measurements
- EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1].freeze
- RAILS_ROOT = Rails.root.to_s
- METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
- PATH_REGEX = /^#{RAILS_ROOT}\/?/
-
- def settings
- @settings ||= {
- enabled: current_application_settings[:metrics_enabled],
- pool_size: current_application_settings[:metrics_pool_size],
- timeout: current_application_settings[:metrics_timeout],
- method_call_threshold: current_application_settings[:metrics_method_call_threshold],
- host: current_application_settings[:metrics_host],
- port: current_application_settings[:metrics_port],
- sample_interval: current_application_settings[:metrics_sample_interval] || 15,
- packet_size: current_application_settings[:metrics_packet_size] || 1
- }
- end
+ # Prometheus histogram buckets used for arbitrary code measurements
+
+ def settings
+ @settings ||= begin
+ current_settings = Gitlab::CurrentSettings.current_application_settings
+
+ {
+ enabled: current_settings[:metrics_enabled],
+ pool_size: current_settings[:metrics_pool_size],
+ timeout: current_settings[:metrics_timeout],
+ method_call_threshold: current_settings[:metrics_method_call_threshold],
+ host: current_settings[:metrics_host],
+ port: current_settings[:metrics_port],
+ sample_interval: current_settings[:metrics_sample_interval] || 15,
+ packet_size: current_settings[:metrics_packet_size] || 1
+ }
+ end
+ end
- def mri?
- RUBY_ENGINE == 'ruby'
- end
+ def mri?
+ RUBY_ENGINE == 'ruby'
+ end
- def method_call_threshold
- # This is memoized since this method is called for every instrumented
- # method. Loading data from an external cache on every method call slows
- # things down too much.
- # in milliseconds
- @method_call_threshold ||= settings[:method_call_threshold]
- end
+ def method_call_threshold
+ # This is memoized since this method is called for every instrumented
+ # method. Loading data from an external cache on every method call slows
+ # things down too much.
+ # in milliseconds
+ @method_call_threshold ||= settings[:method_call_threshold]
+ end
- def submit_metrics(metrics)
- prepared = prepare_metrics(metrics)
+ def submit_metrics(metrics)
+ prepared = prepare_metrics(metrics)
- pool&.with do |connection|
- prepared.each_slice(settings[:packet_size]) do |slice|
- begin
- connection.write_points(slice)
- rescue StandardError
+ pool&.with do |connection|
+ prepared.each_slice(settings[:packet_size]) do |slice|
+ begin
+ connection.write_points(slice)
+ rescue StandardError
+ end
end
end
+ rescue Errno::EADDRNOTAVAIL, SocketError => ex
+ Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
+ Gitlab::EnvironmentLogger.error(ex)
end
- rescue Errno::EADDRNOTAVAIL, SocketError => ex
- Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
- Gitlab::EnvironmentLogger.error(ex)
- end
- def prepare_metrics(metrics)
- metrics.map do |hash|
- new_hash = hash.symbolize_keys
+ def prepare_metrics(metrics)
+ metrics.map do |hash|
+ new_hash = hash.symbolize_keys
- new_hash[:tags].each do |key, value|
- if value.blank?
- new_hash[:tags].delete(key)
- else
- new_hash[:tags][key] = escape_value(value)
+ new_hash[:tags].each do |key, value|
+ if value.blank?
+ new_hash[:tags].delete(key)
+ else
+ new_hash[:tags][key] = escape_value(value)
+ end
end
+
+ new_hash
end
+ end
- new_hash
+ def escape_value(value)
+ value.to_s.gsub('=', '\\=')
end
- end
- def escape_value(value)
- value.to_s.gsub('=', '\\=')
- end
+ # Measures the execution time of a block.
+ #
+ # Example:
+ #
+ # Gitlab::Metrics.measure(:find_by_username_duration) do
+ # User.find_by_username(some_username)
+ # end
+ #
+ # name - The name of the field to store the execution time in.
+ #
+ # Returns the value yielded by the supplied block.
+ def measure(name)
+ trans = current_transaction
+
+ return yield unless trans
+
+ real_start = Time.now.to_f
+ cpu_start = System.cpu_time
+
+ retval = yield
+
+ cpu_stop = System.cpu_time
+ real_stop = Time.now.to_f
+
+ real_time = (real_stop - real_start)
+ cpu_time = cpu_stop - cpu_start
+
+ real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
+ docstring "Measure #{name}"
+ base_labels Transaction::BASE_LABELS
+ buckets EXECUTION_MEASUREMENT_BUCKETS
+ end
- # Measures the execution time of a block.
- #
- # Example:
- #
- # Gitlab::Metrics.measure(:find_by_username_duration) do
- # User.find_by_username(some_username)
- # end
- #
- # name - The name of the field to store the execution time in.
- #
- # Returns the value yielded by the supplied block.
- def measure(name)
- trans = current_transaction
-
- return yield unless trans
-
- real_start = Time.now.to_f
- cpu_start = System.cpu_time
-
- retval = yield
-
- cpu_stop = System.cpu_time
- real_stop = Time.now.to_f
-
- real_time = (real_stop - real_start)
- cpu_time = cpu_stop - cpu_start
-
- Gitlab::Metrics.histogram("gitlab_#{name}_real_duration_seconds".to_sym,
- "Measure #{name}",
- Transaction::BASE_LABELS,
- EXECUTION_MEASUREMENT_BUCKETS)
- .observe(trans.labels, real_time)
-
- Gitlab::Metrics.histogram("gitlab_#{name}_cpu_duration_seconds".to_sym,
- "Measure #{name}",
- Transaction::BASE_LABELS,
- EXECUTION_MEASUREMENT_BUCKETS)
- .observe(trans.labels, cpu_time / 1000.0)
-
- # InfluxDB stores the _real_time time values as milliseconds
- trans.increment("#{name}_real_time", real_time * 1000, false)
- trans.increment("#{name}_cpu_time", cpu_time, false)
- trans.increment("#{name}_call_count", 1, false)
-
- retval
- end
+ real_duration_seconds.observe(trans.labels, real_time)
- # Sets the action of the current transaction (if any)
- #
- # action - The name of the action.
- def action=(action)
- trans = current_transaction
+ cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
+ docstring "Measure #{name}"
+ base_labels Transaction::BASE_LABELS
+ buckets EXECUTION_MEASUREMENT_BUCKETS
+ with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
+ end
+ cpu_duration_seconds.observe(trans.labels, cpu_time)
- trans&.action = action
- end
+ # InfluxDB stores the _real_time and _cpu_time time values as milliseconds
+ trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
+ trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
+ trans.increment("#{name}_call_count", 1, false)
- # Tracks an event.
- #
- # See `Gitlab::Metrics::Transaction#add_event` for more details.
- def add_event(*args)
- trans = current_transaction
+ retval
+ end
- trans&.add_event(*args)
- end
+ # Sets the action of the current transaction (if any)
+ #
+ # action - The name of the action.
+ def action=(action)
+ trans = current_transaction
- # Returns the prefix to use for the name of a series.
- def series_prefix
- @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
- end
+ trans&.action = action
+ end
- # Allow access from other metrics related middlewares
- def current_transaction
- Transaction.current
- end
+ # Tracks an event.
+ #
+ # See `Gitlab::Metrics::Transaction#add_event` for more details.
+ def add_event(*args)
+ trans = current_transaction
- # When enabled this should be set before being used as the usual pattern
- # "@foo ||= bar" is _not_ thread-safe.
- # rubocop:disable Gitlab/ModuleWithInstanceVariables
- def pool
- if influx_metrics_enabled?
- if @pool.nil?
- MUTEX.synchronize do
- @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
- host = settings[:host]
- port = settings[:port]
-
- InfluxDB::Client
- .new(udp: { host: host, port: port })
+ trans&.add_event(*args)
+ end
+
+ # Returns the prefix to use for the name of a series.
+ def series_prefix
+ @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
+ end
+
+ # Allow access from other metrics related middlewares
+ def current_transaction
+ Transaction.current
+ end
+
+ # When enabled this should be set before being used as the usual pattern
+ # "@foo ||= bar" is _not_ thread-safe.
+ # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ def pool
+ if influx_metrics_enabled?
+ if @pool.nil?
+ MUTEX.synchronize do
+ @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
+ host = settings[:host]
+ port = settings[:port]
+
+ InfluxDB::Client
+ .new(udp: { host: host, port: port })
+ end
end
end
- end
- @pool
+ @pool
+ end
end
+ # rubocop:enable Gitlab/ModuleWithInstanceVariables
end
- # rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index c2f9db56824..b11520a79bb 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -4,26 +4,15 @@ module Gitlab
module Metrics
# Class for tracking timing information about method calls
class MethodCall
- @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false)
- @@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i)
- MUTEX = Mutex.new
+ include Gitlab::Metrics::Methods
BASE_LABELS = { module: nil, method: nil }.freeze
attr_reader :real_time, :cpu_time, :call_count, :labels
- def self.call_duration_histogram
- return @call_duration_histogram if @call_duration_histogram
-
- MUTEX.synchronize do
- @call_duration_histogram ||= Gitlab::Metrics.histogram(
- :gitlab_method_call_duration_seconds,
- 'Method calls real duration',
- Transaction::BASE_LABELS.merge(BASE_LABELS),
- [0.01, 0.05, 0.1, 0.5, 1])
- end
- end
-
- def self.measurement_enabled_cache_expires_at
- @@measurement_enabled_cache_expires_at
+ define_histogram :gitlab_method_call_duration_seconds do
+ docstring 'Method calls real duration'
+ base_labels Transaction::BASE_LABELS.merge(BASE_LABELS)
+ buckets [0.01, 0.05, 0.1, 0.5, 1]
+ with_feature :prometheus_metrics_method_instrumentation
end
# name - The full name of the method (including namespace) such as
@@ -53,8 +42,8 @@ module Gitlab
@cpu_time += cpu_time
@call_count += 1
- if call_measurement_enabled? && above_threshold?
- self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time)
+ if above_threshold?
+ self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time)
end
retval
@@ -78,17 +67,6 @@ module Gitlab
def above_threshold?
real_time.in_milliseconds >= Metrics.method_call_threshold
end
-
- def call_measurement_enabled?
- expires_at = @@measurement_enabled_cache_expires_at.value
- if expires_at < Time.now.to_i
- if @@measurement_enabled_cache_expires_at.compare_and_set(expires_at, 1.minute.from_now.to_i)
- @@measurement_enabled_cache.value = Feature.get(:prometheus_metrics_method_instrumentation).enabled?
- end
- end
-
- @@measurement_enabled_cache.value
- end
end
end
end
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
new file mode 100644
index 00000000000..cd7c1e507f7
--- /dev/null
+++ b/lib/gitlab/metrics/methods.rb
@@ -0,0 +1,129 @@
+# rubocop:disable Style/ClassVars
+
+module Gitlab
+ module Metrics
+ module Methods
+ extend ActiveSupport::Concern
+
+ included do
+ @@_metric_provider_mutex ||= Mutex.new
+ @@_metrics_provider_cache = {}
+ end
+
+ class_methods do
+ def reload_metric!(name)
+ @@_metrics_provider_cache.delete(name)
+ end
+
+ private
+
+ def define_metric(type, name, opts = {}, &block)
+ if respond_to?(name)
+ raise ArgumentError, "method #{name} already exists"
+ end
+
+ define_singleton_method(name) do
+ # inlining fetch_metric method to avoid method call overhead when instrumenting hot spots
+ @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block)
+ end
+ end
+
+ def fetch_metric(type, name, opts = {}, &block)
+ @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block)
+ end
+
+ def init_metric(type, name, opts = {}, &block)
+ options = MetricOptions.new(opts)
+ options.evaluate(&block)
+
+ if disabled_by_feature(options)
+ synchronized_cache_fill(name) { NullMetric.instance }
+ else
+ synchronized_cache_fill(name) { build_metric!(type, name, options) }
+ end
+ end
+
+ def synchronized_cache_fill(key)
+ @@_metric_provider_mutex.synchronize do
+ @@_metrics_provider_cache[key] ||= yield
+ end
+ end
+
+ def disabled_by_feature(options)
+ options.with_feature && !Feature.get(options.with_feature).enabled?
+ end
+
+ def build_metric!(type, name, options)
+ case type
+ when :gauge
+ Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode)
+ when :counter
+ Gitlab::Metrics.counter(name, options.docstring, options.base_labels)
+ when :histogram
+ Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets)
+ when :summary
+ raise NotImplementedError, "summary metrics are not currently supported"
+ else
+ raise ArgumentError, "uknown metric type #{type}"
+ end
+ end
+
+ # Fetch and/or initialize counter metric
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def fetch_counter(name, opts = {}, &block)
+ fetch_metric(:counter, name, opts, &block)
+ end
+
+ # Fetch and/or initialize gauge metric
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def fetch_gauge(name, opts = {}, &block)
+ fetch_metric(:gauge, name, opts, &block)
+ end
+
+ # Fetch and/or initialize histogram metric
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def fetch_histogram(name, opts = {}, &block)
+ fetch_metric(:histogram, name, opts, &block)
+ end
+
+ # Fetch and/or initialize summary metric
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def fetch_summary(name, opts = {}, &block)
+ fetch_metric(:summary, name, opts, &block)
+ end
+
+ # Define metric accessor method for a Counter
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def define_counter(name, opts = {}, &block)
+ define_metric(:counter, name, opts, &block)
+ end
+
+ # Define metric accessor method for a Gauge
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def define_gauge(name, opts = {}, &block)
+ define_metric(:gauge, name, opts, &block)
+ end
+
+ # Define metric accessor method for a Histogram
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def define_histogram(name, opts = {}, &block)
+ define_metric(:histogram, name, opts, &block)
+ end
+
+ # Define metric accessor method for a Summary
+ # @param [Symbol] name
+ # @param [Hash] opts
+ def define_summary(name, opts = {}, &block)
+ define_metric(:summary, name, opts, &block)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb
new file mode 100644
index 00000000000..70e122d4e15
--- /dev/null
+++ b/lib/gitlab/metrics/methods/metric_options.rb
@@ -0,0 +1,61 @@
+module Gitlab
+ module Metrics
+ module Methods
+ class MetricOptions
+ SMALL_NETWORK_BUCKETS = [0.005, 0.01, 0.1, 1, 10].freeze
+
+ def initialize(options = {})
+ @multiprocess_mode = options[:multiprocess_mode] || :all
+ @buckets = options[:buckets] || SMALL_NETWORK_BUCKETS
+ @base_labels = options[:base_labels] || {}
+ @docstring = options[:docstring]
+ @with_feature = options[:with_feature]
+ end
+
+ # Documentation describing metric in metrics endpoint '/-/metrics'
+ def docstring(docstring = nil)
+ @docstring = docstring unless docstring.nil?
+
+ @docstring
+ end
+
+ # Gauge aggregation mode for multiprocess metrics
+ # - :all (default) returns each gauge for every process
+ # - :livesum all process'es gauges summed up
+ # - :max maximum value of per process gauges
+ # - :min minimum value of per process gauges
+ def multiprocess_mode(mode = nil)
+ @multiprocess_mode = mode unless mode.nil?
+
+ @multiprocess_mode
+ end
+
+ # Measurement buckets for histograms
+ def buckets(buckets = nil)
+ @buckets = buckets unless buckets.nil?
+
+ @buckets
+ end
+
+ # Base labels are merged with per metric labels
+ def base_labels(base_labels = nil)
+ @base_labels = base_labels unless base_labels.nil?
+
+ @base_labels
+ end
+
+ # Use feature toggle to control whether certain metric is enabled/disabled
+ def with_feature(name = nil)
+ @with_feature = name unless name.nil?
+
+ @with_feature
+ end
+
+ def evaluate(&block)
+ instance_eval(&block) if block_given?
+ self
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb
index 3b5a2907195..aabada5c21a 100644
--- a/lib/gitlab/metrics/null_metric.rb
+++ b/lib/gitlab/metrics/null_metric.rb
@@ -2,6 +2,8 @@ module Gitlab
module Metrics
# Mocks ::Prometheus::Client::Metric and all derived metrics
class NullMetric
+ include Singleton
+
def method_missing(name, *args, &block)
nil
end
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
index b0b8e8436db..d12ba0ec176 100644
--- a/lib/gitlab/metrics/prometheus.rb
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -3,73 +3,76 @@ require 'prometheus/client'
module Gitlab
module Metrics
module Prometheus
- include Gitlab::CurrentSettings
- include Gitlab::Utils::StrongMemoize
+ extend ActiveSupport::Concern
REGISTRY_MUTEX = Mutex.new
PROVIDER_MUTEX = Mutex.new
- def metrics_folder_present?
- multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir
+ class_methods do
+ include Gitlab::Utils::StrongMemoize
- multiprocess_files_dir &&
- ::Dir.exist?(multiprocess_files_dir) &&
- ::File.writable?(multiprocess_files_dir)
- end
+ def metrics_folder_present?
+ multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir
- def prometheus_metrics_enabled?
- strong_memoize(:prometheus_metrics_enabled) do
- prometheus_metrics_enabled_unmemoized
+ multiprocess_files_dir &&
+ ::Dir.exist?(multiprocess_files_dir) &&
+ ::File.writable?(multiprocess_files_dir)
+ end
+
+ def prometheus_metrics_enabled?
+ strong_memoize(:prometheus_metrics_enabled) do
+ prometheus_metrics_enabled_unmemoized
+ end
end
- end
- def registry
- strong_memoize(:registry) do
- REGISTRY_MUTEX.synchronize do
- strong_memoize(:registry) do
- ::Prometheus::Client.registry
+ def registry
+ strong_memoize(:registry) do
+ REGISTRY_MUTEX.synchronize do
+ strong_memoize(:registry) do
+ ::Prometheus::Client.registry
+ end
end
end
end
- end
- def counter(name, docstring, base_labels = {})
- safe_provide_metric(:counter, name, docstring, base_labels)
- end
+ def counter(name, docstring, base_labels = {})
+ safe_provide_metric(:counter, name, docstring, base_labels)
+ end
- def summary(name, docstring, base_labels = {})
- safe_provide_metric(:summary, name, docstring, base_labels)
- end
+ def summary(name, docstring, base_labels = {})
+ safe_provide_metric(:summary, name, docstring, base_labels)
+ end
- def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all)
- safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode)
- end
+ def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all)
+ safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode)
+ end
- def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
- safe_provide_metric(:histogram, name, docstring, base_labels, buckets)
- end
+ def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
+ safe_provide_metric(:histogram, name, docstring, base_labels, buckets)
+ end
- private
+ private
- def safe_provide_metric(method, name, *args)
- metric = provide_metric(name)
- return metric if metric
+ def safe_provide_metric(method, name, *args)
+ metric = provide_metric(name)
+ return metric if metric
- PROVIDER_MUTEX.synchronize do
- provide_metric(name) || registry.method(method).call(name, *args)
+ PROVIDER_MUTEX.synchronize do
+ provide_metric(name) || registry.method(method).call(name, *args)
+ end
end
- end
- def provide_metric(name)
- if prometheus_metrics_enabled?
- registry.get(name)
- else
- NullMetric.new
+ def provide_metric(name)
+ if prometheus_metrics_enabled?
+ registry.get(name)
+ else
+ NullMetric.instance
+ end
end
- end
- def prometheus_metrics_enabled_unmemoized
- metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false
+ def prometheus_metrics_enabled_unmemoized
+ metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false
+ end
end
end
end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 3da474fc1ec..b600e8a2a50 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -3,6 +3,14 @@ module Gitlab
module Subscribers
# Class for tracking the rendering timings of views.
class ActionView < ActiveSupport::Subscriber
+ include Gitlab::Metrics::Methods
+ define_histogram :gitlab_view_rendering_duration_seconds do
+ docstring 'View rendering time'
+ base_labels Transaction::BASE_LABELS.merge({ path: nil })
+ buckets [0.001, 0.01, 0.1, 1, 10.0]
+ with_feature :prometheus_metrics_view_instrumentation
+ end
+
attach_to :action_view
SERIES = 'views'.freeze
@@ -15,30 +23,18 @@ module Gitlab
private
- def metric_view_rendering_duration_seconds
- @metric_view_rendering_duration_seconds ||= Gitlab::Metrics.histogram(
- :gitlab_view_rendering_duration_seconds,
- 'View rendering time',
- Transaction::BASE_LABELS.merge({ path: nil }),
- [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0]
- )
- end
-
def track(event)
values = values_for(event)
tags = tags_for(event)
- metric_view_rendering_duration_seconds.observe(
- current_transaction.labels.merge(tags),
- event.duration
- )
+ self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration)
current_transaction.increment(:view_duration, event.duration)
current_transaction.add_metric(SERIES, values, tags)
end
def relative_path(path)
- path.gsub(/^#{Rails.root.to_s}\/?/, '')
+ path.gsub(%r{^#{Rails.root.to_s}/?}, '')
end
def values_for(event)
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index ead1acb8d44..4b3e8d0a6a0 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -3,12 +3,13 @@ module Gitlab
module Subscribers
# Class for tracking the total query duration of a transaction.
class ActiveRecord < ActiveSupport::Subscriber
+ include Gitlab::Metrics::Methods
attach_to :active_record
def sql(event)
return unless current_transaction
- metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
+ self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
current_transaction.increment(:sql_duration, event.duration, false)
current_transaction.increment(:sql_count, 1, false)
@@ -16,17 +17,14 @@ module Gitlab
private
- def current_transaction
- Transaction.current
+ define_histogram :gitlab_sql_duration_seconds do
+ docstring 'SQL time'
+ base_labels Transaction::BASE_LABELS
+ buckets [0.001, 0.01, 0.1, 1.0, 10.0]
end
- def metric_sql_duration_seconds
- @metric_sql_duration_seconds ||= Gitlab::Metrics.histogram(
- :gitlab_sql_duration_seconds,
- 'SQL time',
- Transaction::BASE_LABELS,
- [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0]
- )
+ def current_transaction
+ Transaction.current
end
end
end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index e7975c023a9..45b9e14ba55 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -2,11 +2,12 @@ module Gitlab
module Metrics
# Class for storing metrics information of a single transaction.
class Transaction
+ include Gitlab::Metrics::Methods
+
# base labels shared among all transactions
BASE_LABELS = { controller: nil, action: nil }.freeze
THREAD_KEY = :_gitlab_metrics_transaction
- METRICS_MUTEX = Mutex.new
# The series to store events (e.g. Git pushes) in.
EVENT_SERIES = 'events'.freeze
@@ -54,8 +55,8 @@ module Gitlab
@memory_after = System.memory_usage
@finished_at = System.monotonic_time
- self.class.metric_transaction_duration_seconds.observe(labels, duration)
- self.class.metric_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0)
+ self.class.gitlab_transaction_duration_seconds.observe(labels, duration)
+ self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0)
Thread.current[THREAD_KEY] = nil
end
@@ -72,7 +73,7 @@ module Gitlab
# event_name - The name of the event (e.g. "git_push").
# tags - A set of tags to attach to the event.
def add_event(event_name, tags = {})
- self.class.metric_event_counter(event_name, tags).increment(tags.merge(labels))
+ self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: tags).increment(tags.merge(labels))
@metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event)
end
@@ -86,12 +87,12 @@ module Gitlab
end
def increment(name, value, use_prometheus = true)
- self.class.metric_transaction_counter(name).increment(labels, value) if use_prometheus
+ self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus
@values[name] += value
end
def set(name, value, use_prometheus = true)
- self.class.metric_transaction_gauge(name).set(labels, value) if use_prometheus
+ self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
@values[name] = value
end
@@ -136,64 +137,28 @@ module Gitlab
"#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty?
end
- def self.metric_transaction_duration_seconds
- return @metric_transaction_duration_seconds if @metric_transaction_duration_seconds
-
- METRICS_MUTEX.synchronize do
- @metric_transaction_duration_seconds ||= Gitlab::Metrics.histogram(
- :gitlab_transaction_duration_seconds,
- 'Transaction duration',
- BASE_LABELS,
- [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0]
- )
- end
- end
-
- def self.metric_transaction_allocated_memory_bytes
- return @metric_transaction_allocated_memory_bytes if @metric_transaction_allocated_memory_bytes
-
- METRICS_MUTEX.synchronize do
- @metric_transaction_allocated_memory_bytes ||= Gitlab::Metrics.histogram(
- :gitlab_transaction_allocated_memory_bytes,
- 'Transaction allocated memory bytes',
- BASE_LABELS,
- [1000, 10000, 20000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 100000000]
- )
- end
+ define_histogram :gitlab_transaction_duration_seconds do
+ docstring 'Transaction duration'
+ base_labels BASE_LABELS
+ buckets [0.001, 0.01, 0.1, 1.0, 10.0]
end
- def self.metric_event_counter(event_name, tags)
- return @metric_event_counters[event_name] if @metric_event_counters&.has_key?(event_name)
-
- METRICS_MUTEX.synchronize do
- @metric_event_counters ||= {}
- @metric_event_counters[event_name] ||= Gitlab::Metrics.counter(
- "gitlab_transaction_event_#{event_name}_total".to_sym,
- "Transaction event #{event_name} counter",
- tags.merge(BASE_LABELS)
- )
- end
- end
-
- def self.metric_transaction_counter(name)
- return @metric_transaction_counters[name] if @metric_transaction_counters&.has_key?(name)
-
- METRICS_MUTEX.synchronize do
- @metric_transaction_counters ||= {}
- @metric_transaction_counters[name] ||= Gitlab::Metrics.counter(
- "gitlab_transaction_#{name}_total".to_sym, "Transaction #{name} counter", BASE_LABELS
- )
- end
+ define_histogram :gitlab_transaction_allocated_memory_bytes do
+ docstring 'Transaction allocated memory bytes'
+ base_labels BASE_LABELS
+ buckets [100, 1000, 10000, 100000, 1000000, 10000000]
+ with_feature :prometheus_metrics_transaction_allocated_memory
end
- def self.metric_transaction_gauge(name)
- return @metric_transaction_gauges[name] if @metric_transaction_gauges&.has_key?(name)
+ def self.transaction_metric(name, type, prefix: nil, tags: {})
+ metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym
+ fetch_metric(type, metric_name) do
+ docstring "Transaction #{prefix}#{name} #{type}"
+ base_labels tags.merge(BASE_LABELS)
- METRICS_MUTEX.synchronize do
- @metric_transaction_gauges ||= {}
- @metric_transaction_gauges[name] ||= Gitlab::Metrics.gauge(
- "gitlab_transaction_#{name}".to_sym, "Transaction gauge #{name}", BASE_LABELS, :livesum
- )
+ if type == :gauge
+ multiprocess_mode :livesum
+ end
end
end
end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index c6a56277922..1a570f480c6 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -4,7 +4,6 @@ module Gitlab
module Middleware
class Go
include ActionView::Helpers::TagHelper
- include Gitlab::CurrentSettings
PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze
@@ -42,7 +41,7 @@ module Gitlab
project_url = URI.join(config.gitlab.url, path)
import_prefix = strip_url(project_url.to_s)
- repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh'
+ repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh'
shell = config.gitlab_shell
port = ":#{shell.ssh_port}" unless shell.ssh_port == 22
"ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git"
@@ -56,12 +55,12 @@ module Gitlab
end
def strip_url(url)
- url.gsub(/\Ahttps?:\/\//, '')
+ url.gsub(%r{\Ahttps?://}, '')
end
def project_path(request)
path_info = request.env["PATH_INFO"]
- path_info.sub!(/^\//, '')
+ path_info.sub!(%r{^/}, '')
project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX)
return unless project_path_match
diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb
index 85ffa8aca68..aa1e9dc0fdb 100644
--- a/lib/gitlab/middleware/static.rb
+++ b/lib/gitlab/middleware/static.rb
@@ -1,7 +1,7 @@
module Gitlab
module Middleware
class Static < ActionDispatch::Static
- UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze
+ UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze
def call(env)
return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index fff9360ea27..e40a001d20c 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -55,7 +55,7 @@ module Gitlab
user ||= find_or_build_ldap_user if auto_link_ldap_user?
user ||= build_new_user if signup_enabled?
- user.external = true if external_provider? && user
+ user.external = true if external_provider? && user&.new_record?
user
end
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index e29e168fc5a..6c2b2036074 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -1,7 +1,5 @@
module Gitlab
module PerformanceBar
- extend Gitlab::CurrentSettings
-
ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze
EXPIRY_TIME = 5.minutes
@@ -13,7 +11,7 @@ module Gitlab
end
def self.allowed_group_id
- current_application_settings.performance_bar_allowed_group_id
+ Gitlab::CurrentSettings.performance_bar_allowed_group_id
end
def self.allowed_user_ids
diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb
index 4780675a492..fe4bdfe3831 100644
--- a/lib/gitlab/polling_interval.rb
+++ b/lib/gitlab/polling_interval.rb
@@ -1,12 +1,10 @@
module Gitlab
class PollingInterval
- extend Gitlab::CurrentSettings
-
HEADER_NAME = 'Poll-Interval'.freeze
def self.set_header(response, interval:)
if polling_enabled?
- multiplier = current_application_settings.polling_interval_multiplier
+ multiplier = Gitlab::CurrentSettings.polling_interval_multiplier
value = (interval * multiplier).to_i
else
value = -1
@@ -16,7 +14,7 @@ module Gitlab
end
def self.polling_enabled?
- !current_application_settings.polling_interval_multiplier.zero?
+ !Gitlab::CurrentSettings.polling_interval_multiplier.zero?
end
end
end
diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb
index 4bc5cda8cb5..b9832a724c4 100644
--- a/lib/gitlab/popen.rb
+++ b/lib/gitlab/popen.rb
@@ -5,7 +5,17 @@ module Gitlab
module Popen
extend self
- def popen(cmd, path = nil, vars = {})
+ Result = Struct.new(:cmd, :stdout, :stderr, :status, :duration)
+
+ # Returns [stdout + stderr, status]
+ def popen(cmd, path = nil, vars = {}, &block)
+ result = popen_with_detail(cmd, path, vars, &block)
+
+ [result.stdout << result.stderr, result.status&.exitstatus]
+ end
+
+ # Returns Result
+ def popen_with_detail(cmd, path = nil, vars = {})
unless cmd.is_a?(Array)
raise "System commands must be given as an array of strings"
end
@@ -18,18 +28,21 @@ module Gitlab
FileUtils.mkdir_p(path)
end
- cmd_output = ""
- cmd_status = 0
+ cmd_stdout = ''
+ cmd_stderr = ''
+ cmd_status = nil
+ start = Time.now
+
Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr|
yield(stdin) if block_given?
stdin.close
- cmd_output << stdout.read
- cmd_output << stderr.read
- cmd_status = wait_thr.value.exitstatus
+ cmd_stdout = stdout.read
+ cmd_stderr = stderr.read
+ cmd_status = wait_thr.value
end
- [cmd_output, cmd_status]
+ Result.new(cmd, cmd_stdout, cmd_stderr, cmd_status, Time.now - start)
end
end
end
diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb
new file mode 100644
index 00000000000..f44035a48bb
--- /dev/null
+++ b/lib/gitlab/popen/runner.rb
@@ -0,0 +1,46 @@
+module Gitlab
+ module Popen
+ class Runner
+ attr_reader :results
+
+ def initialize
+ @results = []
+ end
+
+ def run(commands, &block)
+ commands.each do |cmd|
+ # yield doesn't support blocks, so we need to use a block variable
+ block.call(cmd) do # rubocop:disable Performance/RedundantBlockCall
+ cmd_result = Gitlab::Popen.popen_with_detail(cmd)
+
+ results << cmd_result
+
+ cmd_result
+ end
+ end
+ end
+
+ def all_success_and_clean?
+ all_success? && all_stderr_empty?
+ end
+
+ def all_success?
+ results.all? { |result| result.status.success? }
+ end
+
+ def all_stderr_empty?
+ results.all? { |result| result.stderr.empty? }
+ end
+
+ def failed_results
+ results.reject { |result| result.status.success? }
+ end
+
+ def warned_results
+ results.select do |result|
+ result.status.success? && !result.stderr.empty?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb
new file mode 100644
index 00000000000..95d94b3cc68
--- /dev/null
+++ b/lib/gitlab/profiler.rb
@@ -0,0 +1,142 @@
+# coding: utf-8
+module Gitlab
+ module Profiler
+ FILTERED_STRING = '[FILTERED]'.freeze
+
+ IGNORE_BACKTRACES = %w[
+ lib/gitlab/i18n.rb
+ lib/gitlab/request_context.rb
+ config/initializers
+ lib/gitlab/database/load_balancing/
+ lib/gitlab/etag_caching/
+ lib/gitlab/metrics/
+ lib/gitlab/middleware/
+ lib/gitlab/performance_bar/
+ lib/gitlab/request_profiler/
+ lib/gitlab/profiler.rb
+ ].freeze
+
+ # Takes a URL to profile (can be a fully-qualified URL, or an absolute path)
+ # and returns the ruby-prof profile result. Formatting that result is the
+ # caller's responsibility. Requests are GET requests unless post_data is
+ # passed.
+ #
+ # Optional arguments:
+ # - logger: will be used for SQL logging, including a summary at the end of
+ # the log file of the total time spent per model class.
+ #
+ # - post_data: a string of raw POST data to use. Changes the HTTP verb to
+ # POST.
+ #
+ # - user: a user to authenticate as. Only works if the user has a valid
+ # personal access token.
+ #
+ # - private_token: instead of providing a user instance, the token can be
+ # given as a string. Takes precedence over the user option.
+ def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil)
+ app = ActionDispatch::Integration::Session.new(Rails.application)
+ verb = :get
+ headers = {}
+
+ if post_data
+ verb = :post
+ headers['Content-Type'] = 'application/json'
+ end
+
+ if user
+ private_token ||= user.personal_access_tokens.active.pluck(:token).first
+ end
+
+ headers['Private-Token'] = private_token if private_token
+ logger = create_custom_logger(logger, private_token: private_token)
+
+ RequestStore.begin!
+
+ # Make an initial call for an asset path in development mode to avoid
+ # sprockets dominating the profiler output.
+ ActionController::Base.helpers.asset_path('katex.css') if Rails.env.development?
+
+ # Rails loads internationalization files lazily the first time a
+ # translation is needed. Running this prevents this overhead from showing
+ # up in profiles.
+ ::I18n.t('.')[:test_string]
+
+ # Remove API route mounting from the profile.
+ app.get('/api/v4/users')
+
+ result = with_custom_logger(logger) do
+ RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend
+ end
+
+ RequestStore.end!
+
+ log_load_times_by_model(logger)
+
+ result
+ end
+
+ def self.create_custom_logger(logger, private_token: nil)
+ return unless logger
+
+ logger.dup.tap do |new_logger|
+ new_logger.instance_variable_set(:@private_token, private_token)
+
+ class << new_logger
+ attr_reader :load_times_by_model, :private_token
+
+ def debug(message, *)
+ message.gsub!(private_token, FILTERED_STRING) if private_token
+
+ _, type, time = *message.match(/(\w+) Load \(([0-9.]+)ms\)/)
+
+ if type && time
+ @load_times_by_model ||= {}
+ @load_times_by_model[type] ||= 0
+ @load_times_by_model[type] += time.to_f
+ end
+
+ super
+
+ backtrace = Rails.backtrace_cleaner.clean(caller)
+
+ backtrace.each do |caller_line|
+ next if caller_line.match(Regexp.union(IGNORE_BACKTRACES))
+
+ stripped_caller_line = caller_line.sub("#{Rails.root}/", '')
+
+ super(" ↳ #{stripped_caller_line}")
+ end
+ end
+ end
+ end
+ end
+
+ def self.with_custom_logger(logger)
+ original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging
+ original_activerecord_logger = ActiveRecord::Base.logger
+ original_actioncontroller_logger = ActionController::Base.logger
+
+ if logger
+ ActiveSupport::LogSubscriber.colorize_logging = false
+ ActiveRecord::Base.logger = logger
+ ActionController::Base.logger = logger
+ end
+
+ result = yield
+
+ ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging
+ ActiveRecord::Base.logger = original_activerecord_logger
+ ActionController::Base.logger = original_actioncontroller_logger
+
+ result
+ end
+
+ def self.log_load_times_by_model(logger)
+ return unless logger.respond_to?(:load_times_by_model)
+
+ logger.load_times_by_model.to_a.sort_by(&:last).reverse.each do |(model, time)|
+ logger.info("#{model} total: #{time.round(2)}ms")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 7771b15069b..4823f703ba4 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -20,7 +20,7 @@ module Gitlab
when 'commits'
Kaminari.paginate_array(commits).page(page).per(per_page)
else
- super
+ super(scope, page, false)
end
end
diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb
index 09fa14764e6..2819c7d062c 100644
--- a/lib/gitlab/protocol_access.rb
+++ b/lib/gitlab/protocol_access.rb
@@ -1,14 +1,12 @@
module Gitlab
module ProtocolAccess
- extend Gitlab::CurrentSettings
-
def self.allowed?(protocol)
if protocol == 'web'
true
- elsif current_application_settings.enabled_git_access_protocol.blank?
+ elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank?
true
else
- protocol == current_application_settings.enabled_git_access_protocol
+ protocol == Gitlab::CurrentSettings.enabled_git_access_protocol
end
end
end
diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb
new file mode 100644
index 00000000000..f64f1757144
--- /dev/null
+++ b/lib/gitlab/query_limiting.rb
@@ -0,0 +1,36 @@
+module Gitlab
+ module QueryLimiting
+ # Returns true if we should enable tracking of query counts.
+ #
+ # This is only enabled in production/staging if we're running on GitLab.com.
+ # This ensures we don't produce any errors that users can't do anything
+ # about themselves.
+ def self.enable?
+ Gitlab.com? || Rails.env.development? || Rails.env.test?
+ end
+
+ # Allows the current request to execute any number of SQL queries.
+ #
+ # This method should _only_ be used when there's a corresponding issue to
+ # reduce the number of queries.
+ #
+ # The issue URL is only meant to push developers into creating an issue
+ # instead of blindly whitelisting offending blocks of code.
+ def self.whitelist(issue_url)
+ return unless enable_whitelist?
+
+ unless issue_url.start_with?('https://')
+ raise(
+ ArgumentError,
+ 'You must provide a valid issue URL in order to whitelist a block of code'
+ )
+ end
+
+ Transaction&.current&.whitelisted = true
+ end
+
+ def self.enable_whitelist?
+ Rails.env.development? || Rails.env.test?
+ end
+ end
+end
diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb
new file mode 100644
index 00000000000..66049c94ec6
--- /dev/null
+++ b/lib/gitlab/query_limiting/active_support_subscriber.rb
@@ -0,0 +1,11 @@
+module Gitlab
+ module QueryLimiting
+ class ActiveSupportSubscriber < ActiveSupport::Subscriber
+ attach_to :active_record
+
+ def sql(*)
+ Transaction.current&.increment
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb
new file mode 100644
index 00000000000..949ae79a047
--- /dev/null
+++ b/lib/gitlab/query_limiting/middleware.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module QueryLimiting
+ # Middleware for reporting (or raising) when a request performs more than a
+ # certain amount of database queries.
+ class Middleware
+ CONTROLLER_KEY = 'action_controller.instance'.freeze
+ ENDPOINT_KEY = 'api.endpoint'.freeze
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ transaction, retval = Transaction.run do
+ @app.call(env)
+ end
+
+ transaction.action = action_name(env)
+ transaction.act_upon_results
+
+ retval
+ end
+
+ def action_name(env)
+ if env[CONTROLLER_KEY]
+ action_for_rails(env)
+ elsif env[ENDPOINT_KEY]
+ action_for_grape(env)
+ end
+ end
+
+ private
+
+ def action_for_rails(env)
+ controller = env[CONTROLLER_KEY]
+ action = "#{controller.class.name}##{controller.action_name}"
+
+ if controller.content_type == 'text/html'
+ action
+ else
+ "#{action} (#{controller.content_type})"
+ end
+ end
+
+ def action_for_grape(env)
+ endpoint = env[ENDPOINT_KEY]
+ route = endpoint.route rescue nil
+
+ "#{route.request_method} #{route.path}" if route
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb
new file mode 100644
index 00000000000..3cbafadb0d0
--- /dev/null
+++ b/lib/gitlab/query_limiting/transaction.rb
@@ -0,0 +1,83 @@
+module Gitlab
+ module QueryLimiting
+ class Transaction
+ THREAD_KEY = :__gitlab_query_counts_transaction
+
+ attr_accessor :count, :whitelisted
+
+ # The name of the action (e.g. `UsersController#show`) that is being
+ # executed.
+ attr_accessor :action
+
+ # The maximum number of SQL queries that can be executed in a request. For
+ # the sake of keeping things simple we hardcode this value here, it's not
+ # supposed to be changed very often anyway.
+ THRESHOLD = 100
+
+ # Error that is raised whenever exceeding the maximum number of queries.
+ ThresholdExceededError = Class.new(StandardError)
+
+ def self.current
+ Thread.current[THREAD_KEY]
+ end
+
+ # Starts a new transaction and returns it and the blocks' return value.
+ #
+ # Example:
+ #
+ # transaction, retval = Transaction.run do
+ # 10
+ # end
+ #
+ # retval # => 10
+ def self.run
+ transaction = new
+ Thread.current[THREAD_KEY] = transaction
+
+ [transaction, yield]
+ ensure
+ Thread.current[THREAD_KEY] = nil
+ end
+
+ def initialize
+ @action = nil
+ @count = 0
+ @whitelisted = false
+ end
+
+ # Sends a notification based on the number of executed SQL queries.
+ def act_upon_results
+ return unless threshold_exceeded?
+
+ error = ThresholdExceededError.new(error_message)
+
+ if raise_error?
+ raise(error)
+ else
+ # Raven automatically logs to the Rails log if disabled, thus we don't
+ # need to manually log anything in case Sentry support is not enabled.
+ Raven.capture_exception(error)
+ end
+ end
+
+ def increment
+ @count += 1 unless whitelisted
+ end
+
+ def raise_error?
+ Rails.env.test?
+ end
+
+ def threshold_exceeded?
+ count > THRESHOLD
+ end
+
+ def error_message
+ header = 'Too many SQL queries were executed'
+ header += " in #{action}" if action
+
+ "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed"
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
index 3f52402b31f..7328c517a30 100644
--- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
+++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
@@ -9,7 +9,7 @@ module Gitlab
# if date doesn't present return time with current date
# in other cases return nil
class SpendTimeAndDateSeparator
- DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/
+ DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}
def initialize(spend_command_arg)
@spend_arg = spend_command_arg
diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb
index c463dd487a0..c9efa28d7e7 100644
--- a/lib/gitlab/recaptcha.rb
+++ b/lib/gitlab/recaptcha.rb
@@ -1,12 +1,10 @@
module Gitlab
module Recaptcha
- extend Gitlab::CurrentSettings
-
def self.load_configurations!
- if current_application_settings.recaptcha_enabled
+ if Gitlab::CurrentSettings.recaptcha_enabled
::Recaptcha.configure do |config|
- config.public_key = current_application_settings.recaptcha_site_key
- config.private_key = current_application_settings.recaptcha_private_key
+ config.public_key = Gitlab::CurrentSettings.recaptcha_site_key
+ config.private_key = Gitlab::CurrentSettings.recaptcha_private_key
end
true
@@ -14,7 +12,7 @@ module Gitlab
end
def self.enabled?
- current_application_settings.recaptcha_enabled
+ Gitlab::CurrentSettings.recaptcha_enabled
end
end
end
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
index 9bf019b72e6..a991933e910 100644
--- a/lib/gitlab/redis/cache.rb
+++ b/lib/gitlab/redis/cache.rb
@@ -1,5 +1,5 @@
# please require all dependencies below:
-require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present?
module Gitlab
module Redis
diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb
index 3591fa9145e..79265cf952d 100644
--- a/lib/gitlab/repo_path.rb
+++ b/lib/gitlab/repo_path.rb
@@ -30,7 +30,7 @@ module Gitlab
raise NotFoundError.new("No known storage path matches #{repo_path.inspect}")
end
- result.sub(/\A\/*/, '')
+ result.sub(%r{\A/*}, '')
end
def self.find_project(project_path)
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 70b639501fd..7362514167f 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -40,19 +40,21 @@ module Gitlab
@default_project_filter = default_project_filter
end
- def objects(scope, page = nil)
- case scope
- when 'projects'
- projects.page(page).per(per_page)
- when 'issues'
- issues.page(page).per(per_page)
- when 'merge_requests'
- merge_requests.page(page).per(per_page)
- when 'milestones'
- milestones.page(page).per(per_page)
- else
- Kaminari.paginate_array([]).page(page).per(per_page)
- end
+ def objects(scope, page = nil, without_count = true)
+ collection = case scope
+ when 'projects'
+ projects.page(page).per(per_page)
+ when 'issues'
+ issues.page(page).per(per_page)
+ when 'merge_requests'
+ merge_requests.page(page).per(per_page)
+ when 'milestones'
+ milestones.page(page).per(per_page)
+ else
+ Kaminari.paginate_array([]).page(page).per(per_page)
+ end
+
+ without_count ? collection.without_count : collection
end
def projects_count
@@ -71,18 +73,46 @@ module Gitlab
@milestones_count ||= milestones.count
end
+ def limited_projects_count
+ @limited_projects_count ||= projects.limit(count_limit).count
+ end
+
+ def limited_issues_count
+ return @limited_issues_count if @limited_issues_count
+
+ # By default getting limited count (e.g. 1000+) is fast on issuable
+ # collections except for issues, where filtering both not confidential
+ # and confidential issues user has access to, is too complex.
+ # It's faster to try to fetch all public issues first, then only
+ # if necessary try to fetch all issues.
+ sum = issues(public_only: true).limit(count_limit).count
+ @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum
+ end
+
+ def limited_merge_requests_count
+ @limited_merge_requests_count ||= merge_requests.limit(count_limit).count
+ end
+
+ def limited_milestones_count
+ @limited_milestones_count ||= milestones.limit(count_limit).count
+ end
+
def single_commit_result?
false
end
+ def count_limit
+ 1001
+ end
+
private
def projects
limit_projects.search(query)
end
- def issues
- issues = IssuesFinder.new(current_user).execute
+ def issues(finder_params = {})
+ issues = IssuesFinder.new(current_user, finder_params).execute
unless default_project_filter
issues = issues.where(project_id: project_ids_relation)
end
@@ -94,13 +124,13 @@ module Gitlab
issues.full_search(query)
end
- issues.order('updated_at DESC')
+ issues.reorder('updated_at DESC')
end
def milestones
milestones = Milestone.where(project_id: project_ids_relation)
milestones = milestones.search(query)
- milestones.order('updated_at DESC')
+ milestones.reorder('updated_at DESC')
end
def merge_requests
@@ -116,7 +146,7 @@ module Gitlab
merge_requests.full_search(query)
end
- merge_requests.order('updated_at DESC')
+ merge_requests.reorder('updated_at DESC')
end
def default_scope
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index 94a481a0f2e..98f005cb61b 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -5,9 +5,15 @@ module DeliverNever
end
end
+module MuteNotifications
+ def new_note(note)
+ end
+end
+
module Gitlab
class Seeder
def self.quiet
+ mute_notifications
mute_mailer
SeedFu.quiet = true
@@ -18,6 +24,10 @@ module Gitlab
puts "\nOK".color(:green)
end
+ def self.mute_notifications
+ NotificationService.prepend(MuteNotifications)
+ end
+
def self.mute_mailer
ActionMailer::MessageDelivery.prepend(DeliverNever)
end
diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb
index 159d0e7952e..4a22fc80f75 100644
--- a/lib/gitlab/sentry.rb
+++ b/lib/gitlab/sentry.rb
@@ -1,9 +1,7 @@
module Gitlab
module Sentry
- extend Gitlab::CurrentSettings
-
def self.enabled?
- Rails.env.production? && current_application_settings.sentry_enabled?
+ Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled?
end
def self.context(current_user = nil)
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index d01213bb6e0..e90a90508a2 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -31,7 +31,7 @@ module Gitlab
storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s }
end
- config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages }
+ config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages }
config[:auth] = { token: 'secret' } if Rails.env.test?
config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb
index 8a3e1a5e5bf..89072b01f2e 100644
--- a/lib/gitlab/sherlock/file_sample.rb
+++ b/lib/gitlab/sherlock/file_sample.rb
@@ -16,7 +16,7 @@ module Gitlab
end
def relative_path
- @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '')
+ @relative_path ||= @file.gsub(%r{^#{Rails.root.to_s}/?}, '')
end
def to_param
diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb
index 687332fc5fc..4c88e33699a 100644
--- a/lib/gitlab/sherlock/middleware.rb
+++ b/lib/gitlab/sherlock/middleware.rb
@@ -2,7 +2,7 @@ module Gitlab
module Sherlock
# Rack middleware used for tracking request metrics.
class Middleware
- CONTENT_TYPES = /text\/html|application\/json/i
+ CONTENT_TYPES = %r{text/html|application/json}i
IGNORE_PATHS = %r{^/sherlock}
diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb
index 948bf5e6528..02ddc3f47eb 100644
--- a/lib/gitlab/sherlock/query.rb
+++ b/lib/gitlab/sherlock/query.rb
@@ -4,7 +4,7 @@ module Gitlab
attr_reader :id, :query, :started_at, :finished_at, :backtrace
# SQL identifiers that should be prefixed with newlines.
- PREFIX_NEWLINE = /
+ PREFIX_NEWLINE = %r{
\s+(FROM
|(LEFT|RIGHT)?INNER\s+JOIN
|(LEFT|RIGHT)?OUTER\s+JOIN
@@ -13,7 +13,7 @@ module Gitlab
|GROUP\s+BY
|ORDER\s+BY
|LIMIT
- |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :<
+ |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :<
# Creates a new Query using a String and a separate Array of bindings.
#
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index b85f70e450e..4f86b3e8f73 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -16,7 +16,7 @@ module Gitlab
when 'snippet_blobs'
snippet_blobs.page(page).per(per_page)
else
- super
+ super(scope, nil, false)
end
end
diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb
index 89ca1298120..545e7c74f7e 100644
--- a/lib/gitlab/ssh_public_key.rb
+++ b/lib/gitlab/ssh_public_key.rb
@@ -21,6 +21,22 @@ module Gitlab
technology(name)&.supported_sizes
end
+ def self.sanitize(key_content)
+ ssh_type, *parts = key_content.strip.split
+
+ return key_content if parts.empty?
+
+ parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index|
+ content << part
+
+ if Gitlab::SSHPublicKey.new(content).valid?
+ break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present
+ elsif parts.size == index + 1 # return original content if we've reached the last element
+ break key_content
+ end
+ end
+ end
+
attr_reader :key_text, :key
# Unqualified MD5 fingerprint for compatibility
@@ -37,23 +53,23 @@ module Gitlab
end
def valid?
- key.present?
+ key.present? && bits && technology.supported_sizes.include?(bits)
end
def type
- technology.name if valid?
+ technology.name if key.present?
end
def bits
- return unless valid?
+ return if key.blank?
case type
when :rsa
- key.n.num_bits
+ key.n&.num_bits
when :dsa
- key.p.num_bits
+ key.p&.num_bits
when :ecdsa
- key.group.order.num_bits
+ key.group.order&.num_bits
when :ed25519
256
else
diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index c1182af1014..34bee6fecbe 100644
--- a/lib/tasks/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -1,6 +1,7 @@
require 'rainbow/ext/string'
require 'gitlab/utils/strong_memoize'
+# rubocop:disable Rails/Output
module Gitlab
TaskFailedError = Class.new(StandardError)
TaskAbortedByUserError = Class.new(StandardError)
@@ -96,11 +97,9 @@ module Gitlab
end
def gid_for(group_name)
- begin
- Etc.getgrnam(group_name).gid
- rescue ArgumentError # no group
- "group #{group_name} doesn't exist"
- end
+ Etc.getgrnam(group_name).gid
+ rescue ArgumentError # no group
+ "group #{group_name} doesn't exist"
end
def gitlab_user
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
index 3b64cb32afa..024be6aca44 100644
--- a/lib/gitlab/upgrader.rb
+++ b/lib/gitlab/upgrader.rb
@@ -1,6 +1,3 @@
-require_relative "popen"
-require_relative "version_info"
-
module Gitlab
class Upgrader
def execute
@@ -52,7 +49,7 @@ module Gitlab
def fetch_git_tags
remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
- remote_tags.split("\n").grep(/tags\/v#{current_version.major}/)
+ remote_tags.split("\n").grep(%r{tags/v#{current_version.major}})
end
def update_commands
diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb
index b5f41240529..7d7400bdabf 100644
--- a/lib/gitlab/uploads_transfer.rb
+++ b/lib/gitlab/uploads_transfer.rb
@@ -1,7 +1,7 @@
module Gitlab
class UploadsTransfer < ProjectTransfer
def root_dir
- File.join(CarrierWave.root, FileUploader.base_dir)
+ FileUploader.root
end
end
end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 2adcc9809b3..9d13d1d781f 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -1,8 +1,6 @@
module Gitlab
class UsageData
class << self
- include Gitlab::CurrentSettings
-
def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data }
end
@@ -19,7 +17,7 @@ module Gitlab
def license_usage_data
usage_data = {
- uuid: current_application_settings.uuid,
+ uuid: Gitlab::CurrentSettings.uuid,
hostname: Gitlab.config.gitlab.host,
version: Gitlab::VERSION,
active_user_count: User.active.count,
@@ -79,9 +77,9 @@ module Gitlab
def features_usage_data_ce
{
- signup: current_application_settings.allow_signup?,
+ signup: Gitlab::CurrentSettings.allow_signup?,
ldap: Gitlab.config.ldap.enabled,
- gravatar: current_application_settings.gravatar_enabled?,
+ gravatar: Gitlab::CurrentSettings.gravatar_enabled?,
omniauth: Gitlab.config.omniauth.enabled,
reply_by_email: Gitlab::IncomingEmail.enabled?,
container_registry: Gitlab.config.registry.enabled,
diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb
index 6ced06a863d..2612208a927 100644
--- a/lib/gitlab/visibility_level.rb
+++ b/lib/gitlab/visibility_level.rb
@@ -5,7 +5,6 @@
#
module Gitlab
module VisibilityLevel
- extend CurrentSettings
extend ActiveSupport::Concern
included do
@@ -58,9 +57,9 @@ module Gitlab
end
def allowed_levels
- restricted_levels = current_application_settings.restricted_visibility_levels
+ restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels
- self.values - restricted_levels
+ self.values - Array(restricted_levels)
end
def closest_allowed_level(target_level)
@@ -81,7 +80,7 @@ module Gitlab
end
def non_restricted_level?(level)
- restricted_levels = current_application_settings.restricted_visibility_levels
+ restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels
if restricted_levels.nil?
true
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 0de183858aa..823df67ea39 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -34,7 +34,10 @@ module Gitlab
feature_enabled = case action.to_s
when 'git_receive_pack'
- Gitlab::GitalyClient.feature_enabled?(:post_receive_pack)
+ Gitlab::GitalyClient.feature_enabled?(
+ :post_receive_pack,
+ status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT
+ )
when 'git_upload_pack'
true
when 'info_refs'
@@ -52,14 +55,14 @@ module Gitlab
def lfs_upload_ok(oid, size)
{
- StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload",
+ StoreLFSPath: LfsObjectUploader.workhorse_upload_path,
LfsOid: oid,
LfsSize: size
}
end
def artifact_upload_ok
- { TempPath: JobArtifactUploader.artifacts_upload_path }
+ { TempPath: JobArtifactUploader.workhorse_upload_path }
end
def send_git_blob(repository, blob)
@@ -144,8 +147,11 @@ module Gitlab
end
def send_artifacts_entry(build, entry)
+ file = build.artifacts_file
+ archive = file.file_storage? ? file.path : file.url
+
params = {
- 'Archive' => build.artifacts_file.path,
+ 'Archive' => archive,
'Entry' => Base64.encode64(entry.to_s)
}
@@ -155,6 +161,18 @@ module Gitlab
]
end
+ def send_url(url, allow_redirects: false)
+ params = {
+ 'URL' => url,
+ 'AllowRedirects' => allow_redirects
+ }
+
+ [
+ SEND_DATA_HEADER,
+ "send-url:#{encode(params)}"
+ ]
+ end
+
def terminal_websocket(terminal)
details = {
'Terminal' => {
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 54f51d9d633..0e27a28ea6e 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -17,6 +17,8 @@
## See installation.md#using-https for additional HTTPS configuration details.
upstream gitlab-workhorse {
+ # Gitlab socket file,
+ # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
}
@@ -110,6 +112,8 @@ server {
error_page 502 /502.html;
error_page 503 /503.html;
location ~ ^/(404|422|500|502|503)\.html$ {
+ # Location to the Gitlab's public directory,
+ # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public.
root /home/git/gitlab/public;
internal;
}
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index ed8131ef24f..8218d68f9ba 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -21,6 +21,8 @@
## See installation.md#using-https for additional HTTPS configuration details.
upstream gitlab-workhorse {
+ # Gitlab socket file,
+ # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket
server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0;
}
@@ -160,6 +162,8 @@ server {
error_page 502 /502.html;
error_page 503 /503.html;
location ~ ^/(404|422|500|502|503)\.html$ {
+ # Location to the Gitlab's public directory,
+ # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public
root /home/git/gitlab/public;
internal;
}
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
index 6ee8c8874ec..44ec888c197 100644
--- a/lib/system_check/app/git_version_check.rb
+++ b/lib/system_check/app/git_version_check.rb
@@ -5,7 +5,7 @@ module SystemCheck
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
- @required_version ||= Gitlab::VersionInfo.new(2, 7, 3)
+ @required_version ||= Gitlab::VersionInfo.new(2, 9, 5)
end
def self.current_version
diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb
index c42ae4fe4c4..914ed794601 100644
--- a/lib/system_check/helpers.rb
+++ b/lib/system_check/helpers.rb
@@ -1,5 +1,3 @@
-require 'tasks/gitlab/task_helpers'
-
module SystemCheck
module Helpers
include ::Gitlab::TaskHelpers
diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake
index 7ad2b2e4d39..4b4881cecb8 100644
--- a/lib/tasks/flay.rake
+++ b/lib/tasks/flay.rake
@@ -1,8 +1,8 @@
desc 'Code duplication analyze via flay'
task :flay do
- output = `bundle exec flay --mass 35 app/ lib/gitlab/`
+ output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}`
- if output.include? "Similar code found"
+ if output.include?("Similar code found") || output.include?("IDENTICAL code found")
puts output
exit 1
end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 2383bcf954b..24e37f6c6cc 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -4,7 +4,7 @@ namespace :gitlab do
namespace :backup do
# Create backup of GitLab system
desc "GitLab | Create a backup of the GitLab system"
- task create: :environment do
+ task create: :gitlab_environment do
warn_user_is_not_gitlab
configure_cron_mode
@@ -25,7 +25,7 @@ namespace :gitlab do
# Restore backup of GitLab system
desc 'GitLab | Restore a previously created backup'
- task restore: :environment do
+ task restore: :gitlab_environment do
warn_user_is_not_gitlab
configure_cron_mode
@@ -73,7 +73,7 @@ namespace :gitlab do
end
namespace :repo do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping repositories ...".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
@@ -84,7 +84,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring repositories ...".color(:blue)
Backup::Repository.new.restore
$progress.puts "done".color(:green)
@@ -92,7 +92,7 @@ namespace :gitlab do
end
namespace :db do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping database ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("db")
@@ -103,7 +103,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring database ... ".color(:blue)
Backup::Database.new.restore
$progress.puts "done".color(:green)
@@ -111,7 +111,7 @@ namespace :gitlab do
end
namespace :builds do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping builds ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("builds")
@@ -122,7 +122,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring builds ... ".color(:blue)
Backup::Builds.new.restore
$progress.puts "done".color(:green)
@@ -130,7 +130,7 @@ namespace :gitlab do
end
namespace :uploads do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping uploads ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
@@ -141,7 +141,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring uploads ... ".color(:blue)
Backup::Uploads.new.restore
$progress.puts "done".color(:green)
@@ -149,7 +149,7 @@ namespace :gitlab do
end
namespace :artifacts do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping artifacts ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
@@ -160,7 +160,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring artifacts ... ".color(:blue)
Backup::Artifacts.new.restore
$progress.puts "done".color(:green)
@@ -168,7 +168,7 @@ namespace :gitlab do
end
namespace :pages do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping pages ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("pages")
@@ -179,7 +179,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring pages ... ".color(:blue)
Backup::Pages.new.restore
$progress.puts "done".color(:green)
@@ -187,7 +187,7 @@ namespace :gitlab do
end
namespace :lfs do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping lfs objects ... ".color(:blue)
if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
@@ -198,7 +198,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring lfs objects ... ".color(:blue)
Backup::Lfs.new.restore
$progress.puts "done".color(:green)
@@ -206,7 +206,7 @@ namespace :gitlab do
end
namespace :registry do
- task create: :environment do
+ task create: :gitlab_environment do
$progress.puts "Dumping container registry images ... ".color(:blue)
if Gitlab.config.registry.enabled
@@ -221,7 +221,7 @@ namespace :gitlab do
end
end
- task restore: :environment do
+ task restore: :gitlab_environment do
$progress.puts "Restoring container registry images ... ".color(:blue)
if Gitlab.config.registry.enabled
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index 31cd6bfe6e1..e05a3aad824 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -1,7 +1,3 @@
-# Temporary hack, until we migrate all checks to SystemCheck format
-require 'system_check'
-require 'system_check/helpers'
-
namespace :gitlab do
desc 'GitLab | Check the configuration of GitLab and its environment'
task check: %w{gitlab:gitlab_shell:check
@@ -12,7 +8,7 @@ namespace :gitlab do
namespace :app do
desc 'GitLab | Check the configuration of the GitLab Rails app'
- task check: :environment do
+ task check: :gitlab_environment do
warn_user_is_not_gitlab
checks = [
@@ -43,7 +39,7 @@ namespace :gitlab do
namespace :gitlab_shell do
desc "GitLab | Check the configuration of GitLab Shell"
- task check: :environment do
+ task check: :gitlab_environment do
warn_user_is_not_gitlab
start_checking "GitLab Shell"
@@ -251,7 +247,7 @@ namespace :gitlab do
namespace :sidekiq do
desc "GitLab | Check the configuration of Sidekiq"
- task check: :environment do
+ task check: :gitlab_environment do
warn_user_is_not_gitlab
start_checking "Sidekiq"
@@ -310,7 +306,7 @@ namespace :gitlab do
namespace :incoming_email do
desc "GitLab | Check the configuration of Reply by email"
- task check: :environment do
+ task check: :gitlab_environment do
warn_user_is_not_gitlab
if Gitlab.config.incoming_email.enabled
@@ -333,7 +329,7 @@ namespace :gitlab do
end
namespace :ldap do
- task :check, [:limit] => :environment do |_, args|
+ task :check, [:limit] => :gitlab_environment do |_, args|
# Only show up to 100 results because LDAP directories can be very big.
# This setting only affects the `rake gitlab:check` script.
args.with_defaults(limit: 100)
@@ -389,7 +385,7 @@ namespace :gitlab do
namespace :repo do
desc "GitLab | Check the integrity of the repositories managed by GitLab"
- task check: :environment do
+ task check: :gitlab_environment do
puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red)
Rake::Task["gitlab:git:fsck"].execute
end
@@ -397,7 +393,7 @@ namespace :gitlab do
namespace :orphans do
desc 'Gitlab | Check for orphaned namespaces and repositories'
- task check: :environment do
+ task check: :gitlab_environment do
warn_user_is_not_gitlab
checks = [
SystemCheck::Orphans::NamespaceCheck,
@@ -408,7 +404,7 @@ namespace :gitlab do
end
desc 'GitLab | Check for orphaned namespaces in the repositories path'
- task check_namespaces: :environment do
+ task check_namespaces: :gitlab_environment do
warn_user_is_not_gitlab
checks = [SystemCheck::Orphans::NamespaceCheck]
@@ -416,7 +412,7 @@ namespace :gitlab do
end
desc 'GitLab | Check for orphaned repositories in the repositories path'
- task check_repositories: :environment do
+ task check_repositories: :gitlab_environment do
warn_user_is_not_gitlab
checks = [SystemCheck::Orphans::RepositoryCheck]
@@ -426,8 +422,8 @@ namespace :gitlab do
namespace :user do
desc "GitLab | Check the integrity of a specific user's repositories"
- task :check_repos, [:username] => :environment do |t, args|
- username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue))
+ task :check_repos, [:username] => :gitlab_environment do |t, args|
+ username = args[:username] || prompt("Check repository integrity for username? ".color(:blue))
user = User.find_by(username: username)
if user
repo_dirs = user.authorized_projects.map do |p|
diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake
index 04d56509ac6..5a53eac0897 100644
--- a/lib/tasks/gitlab/cleanup.rake
+++ b/lib/tasks/gitlab/cleanup.rake
@@ -1,9 +1,11 @@
+# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954
+#
namespace :gitlab do
namespace :cleanup do
HASHED_REPOSITORY_NAME = '@hashed'.freeze
desc "GitLab | Cleanup | Clean namespaces"
- task dirs: :environment do
+ task dirs: :gitlab_environment do
warn_user_is_not_gitlab
remove_flag = ENV['REMOVE']
@@ -47,7 +49,7 @@ namespace :gitlab do
end
desc "GitLab | Cleanup | Clean repositories"
- task repos: :environment do
+ task repos: :gitlab_environment do
warn_user_is_not_gitlab
move_suffix = "+orphaned+#{Time.now.to_i}"
@@ -76,7 +78,7 @@ namespace :gitlab do
end
desc "GitLab | Cleanup | Block users that have been removed in LDAP"
- task block_removed_ldap_users: :environment do
+ task block_removed_ldap_users: :gitlab_environment do
warn_user_is_not_gitlab
block_flag = ENV['BLOCK']
@@ -107,7 +109,7 @@ namespace :gitlab do
# released. So likely this should only be run once on gitlab.com
# Faulty refs are moved so they are kept around, else some features break.
desc 'GitLab | Cleanup | Remove faulty deployment refs'
- task move_faulty_deployment_refs: :environment do
+ task move_faulty_deployment_refs: :gitlab_environment do
projects = Project.where(id: Deployment.select(:project_id).distinct)
projects.find_each do |project|
diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index 3f5dd2ae3b3..cb4f7e5c8a8 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :git do
desc "GitLab | Git | Repack"
- task repack: :environment do
+ task repack: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo")
if failures.empty?
puts "Done".color(:green)
@@ -11,7 +11,7 @@ namespace :gitlab do
end
desc "GitLab | Git | Run garbage collection on all repos"
- task gc: :environment do
+ task gc: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting")
if failures.empty?
puts "Done".color(:green)
@@ -21,7 +21,7 @@ namespace :gitlab do
end
desc "GitLab | Git | Prune all repos"
- task prune: :environment do
+ task prune: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune")
if failures.empty?
puts "Done".color(:green)
@@ -31,7 +31,7 @@ namespace :gitlab do
end
desc 'GitLab | Git | Check all repos integrity'
- task fsck: :environment do
+ task fsck: :gitlab_environment do
failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo|
check_config_lock(repo)
check_ref_locks(repo)
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index a2e68c0471b..107ff1d8aeb 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :gitaly do
desc "GitLab | Install or upgrade gitaly"
- task :install, [:dir, :repo] => :environment do |t, args|
+ task :install, [:dir, :repo] => :gitlab_environment do |t, args|
require 'toml'
warn_user_is_not_gitlab
@@ -21,7 +21,11 @@ namespace :gitlab do
_, status = Gitlab::Popen.popen(%w[which gmake])
command << (status.zero? ? 'gmake' : 'make')
- command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test?
+ if Rails.env.test?
+ command.push(
+ 'BUNDLE_FLAGS=--no-deployment',
+ "BUNDLE_PATH=#{Bundler.bundle_path}")
+ end
Gitlab::SetupHelper.create_gitaly_configuration(args.dir)
Dir.chdir(args.dir) do
diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake
index b0a24790c4a..14d1125a03d 100644
--- a/lib/tasks/gitlab/helpers.rake
+++ b/lib/tasks/gitlab/helpers.rake
@@ -1,8 +1,6 @@
-require 'tasks/gitlab/task_helpers'
-
# Prevent StateMachine warnings from outputting during a cron task
StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON']
-namespace :gitlab do
+task gitlab_environment: :environment do
extend SystemCheck::Helpers
end
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index e9fb6a008b0..45e9a1a1c72 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :env do
desc "GitLab | Show information about GitLab and its environment"
- task info: :environment do
+ task info: :gitlab_environment do
# check if there is an RVM environment
rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s)
# check Ruby version
diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake
index 05fcb8e3da5..1d903c81358 100644
--- a/lib/tasks/gitlab/setup.rake
+++ b/lib/tasks/gitlab/setup.rake
@@ -1,6 +1,6 @@
namespace :gitlab do
desc "GitLab | Setup production application"
- task setup: :environment do
+ task setup: :gitlab_environment do
setup_db
end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 12ae4199b69..844664b12d4 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :shell do
desc "GitLab | Install or upgrade gitlab-shell"
- task :install, [:repo] => :environment do |t, args|
+ task :install, [:repo] => :gitlab_environment do |t, args|
warn_user_is_not_gitlab
default_version = Gitlab::Shell.version_required
@@ -58,12 +58,12 @@ namespace :gitlab do
end
desc "GitLab | Setup gitlab-shell"
- task setup: :environment do
+ task setup: :gitlab_environment do
setup
end
desc "GitLab | Build missing projects"
- task build_missing_projects: :environment do
+ task build_missing_projects: :gitlab_environment do
Project.find_each(batch_size: 1000) do |project|
path_to_repo = project.repository.path_to_repo
if File.exist?(path_to_repo)
@@ -80,7 +80,7 @@ namespace :gitlab do
end
desc 'Create or repair repository hooks symlink'
- task create_hooks: :environment do
+ task create_hooks: :gitlab_environment do
warn_user_is_not_gitlab
puts 'Creating/Repairing hooks symlinks for all repositories'
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index 308ffb0e284..b917a293095 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -1,7 +1,7 @@
namespace :gitlab do
namespace :workhorse do
desc "GitLab | Install or upgrade gitlab-workhorse"
- task :install, [:dir, :repo] => :environment do |t, args|
+ task :install, [:dir, :repo] => :gitlab_environment do |t, args|
warn_user_is_not_gitlab
unless args.dir.present?
diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake
index ad2d034b0b4..5c0cc4990fc 100644
--- a/lib/tasks/haml-lint.rake
+++ b/lib/tasks/haml-lint.rake
@@ -2,5 +2,14 @@ unless Rails.env.production?
require 'haml_lint/rake_task'
require 'haml_lint/inline_javascript'
+ # Workaround for warnings from parser/current
+ # TODO: Remove this after we update parser gem
+ task :haml_lint do
+ require 'parser'
+ def Parser.warn(*args)
+ puts(*args) # static-analysis ignores stdout if status is 0
+ end
+ end
+
HamlLint::RakeTask.new
end
diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake
index c996537cfbe..31cbd651edb 100644
--- a/lib/tasks/migrate/setup_postgresql.rake
+++ b/lib/tasks/migrate/setup_postgresql.rake
@@ -1,16 +1,14 @@
-require Rails.root.join('lib/gitlab/database')
-require Rails.root.join('lib/gitlab/database/migration_helpers')
-require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
-require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
-require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
-require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
-require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
-require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
-require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
-require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb')
-
desc 'GitLab | Sets up PostgreSQL'
task setup_postgresql: :environment do
+ require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes')
+ require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes')
+ require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes')
+ require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like')
+ require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes')
+ require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like')
+ require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb')
+ require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb')
+
NamespacesProjectsPathLowerIndexes.new.up
AddUsersLowerUsernameEmailIndexes.new.up
AddLowerPathIndexToRoutes.new.up