diff options
Diffstat (limited to 'lib')
59 files changed, 1115 insertions, 625 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e3258fdcffe..ded5c65e303 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -100,6 +100,8 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :import_status + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :avatar_url do |user, options| user.avatar_url(only_path: false) end @@ -675,6 +677,7 @@ module API class Variable < Grape::Entity expose :key, :value + expose :protected?, as: :protected end class Pipeline < PipelineBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d61450f8258..2c73a6fdc4e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -158,7 +158,7 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) + if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end @@ -311,6 +311,16 @@ module API end end + def present_artifacts!(artifacts_file) + return not_found! unless artifacts_file.exists? + + if artifacts_file.file_storage? + present_file!(artifacts_file.path, artifacts_file.filename) + else + redirect_to(artifacts_file.url) + end + end + private def private_token diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 0223957fde1..8a67de10bca 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -224,16 +224,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d00d4fe1737..deac3934d57 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -109,7 +109,7 @@ module API end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -129,6 +129,7 @@ module API params do requires :name, type: String, desc: 'The name of the project' requires :user_id, type: Integer, desc: 'The ID of a user' + optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' use :optional_params use :create_params @@ -247,7 +248,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 6fbb02cb3aa..4552115b3e2 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -141,7 +141,7 @@ module API patch '/:id/trace' do job = authenticate_job! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -241,16 +241,7 @@ module API get '/:id/artifacts' do job = authenticate_job! - artifacts_file = job.artifacts_file - unless artifacts_file.file_storage? - return redirect_to job.artifacts_file.url - end - - unless artifacts_file.exists? - not_found! - end - - present_file!(artifacts_file.path, artifacts_file.filename) + present_artifacts!(job.artifacts_file) end end end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 05b4b490e27..df4632346dd 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,7 +5,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' + declared_params.key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/users.rb b/lib/api/users.rb index 3d83720b7b9..2070dbd8bc7 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -286,13 +286,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserWorker.perform_async(current_user.id, user.id) + DeleteUserWorker.perform_async(current_user.id, user.id, hard_delete: params[:hard_delete]) end desc 'Block a user. Available only for admins.' diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 21935922414..93ad9eb26b8 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -225,16 +225,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 896c00b88e7..20976b9dd08 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -44,7 +44,7 @@ module API end def set_only_allow_merge_if_pipeline_succeeds! - if params.has_key?(:only_allow_merge_if_build_succeeds) + if params.key?(:only_allow_merge_if_build_succeeds) params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) end end diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb index 81ae4e8137d..d5b90e435ba 100644 --- a/lib/api/v3/time_tracking_endpoints.rb +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -6,7 +6,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.key?(:issue_id) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 5acde41551b..381c4ef50b0 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -42,6 +42,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) @@ -59,13 +60,14 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - if variable.update(value: params[:value]) + if variable.update(declared_params(include_missing: false).except(:key)) present variable, with: Entities::Variable else render_validation_error!(variable) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 51fa3867e67..1f4bda6f588 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.artifacts_path) + super('artifacts', ArtifactUploader.local_artifacts_store) end def create_files_dir diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 4f8efe03bae..c52acbc3ddc 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -22,11 +22,11 @@ module Bitbucket end def inline? - raw.has_key?('inline') + raw.key?('inline') end def has_parent? - raw.has_key?('parent') + raw.key?('parent') end private diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 67b269b330c..e2e91ce99cd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -88,7 +88,7 @@ module Ci patch ":id/trace.txt" do build = authenticate_build! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -187,14 +187,14 @@ module Ci build = authenticate_build! artifacts_file = build.artifacts_file - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - unless artifacts_file.exists? not_found! end + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + present_file!(artifacts_file.path, artifacts_file.filename) end diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb index 2700a5a2ad5..05994bee79d 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -45,9 +45,9 @@ module Gitlab end def format_response(response) - response[:text] = format(response[:text]) if response.has_key?(:text) + response[:text] = format(response[:text]) if response.key?(:text) - if response.has_key?(:attachments) + if response.key?(:attachments) response[:attachments].each do |attachment| attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] attachment[:text] = format(attachment[:text]) if attachment[:text] diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fe..9e14b35b0f8 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -19,7 +19,7 @@ module Gitlab settings = ::ApplicationSetting.last end - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + settings ||= ::ApplicationSetting.create_from_defaults end settings || in_memory_application_settings @@ -46,7 +46,8 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + ActiveRecord::Base.connection.table_exists?('application_settings') && + !ActiveRecord::Migrator.needs_migration? rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..dbe28e6bb93 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index d137cc1bae6..ca49eda51fb 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -9,9 +9,11 @@ module Gitlab # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route # - Ending in `issues/id`/realtime_changes` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests new].freeze + commit pipelines merge_requests builds + new environments].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) + ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), @@ -40,6 +42,14 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z), + 'project_build' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), + 'environments' ) ].freeze diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 6b0a66365a7..d60e607b02b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 297531db4cc..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 6e85b14d5db..8926aa19925 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff @@ -201,7 +201,7 @@ module Gitlab prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta init_from_rugged(raw_diff) - when Gitaly::CommitDiffResponse + when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible when Gitaly::CommitDelta @@ -302,15 +302,15 @@ module Gitlab end end - def init_from_gitaly(msg) - @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) - @new_path = encode!(msg.to_path.dup) - @old_path = encode!(msg.from_path.dup) - @a_mode = msg.old_mode.to_s(8) - @b_mode = msg.new_mode.to_s(8) - @new_file = msg.from_id == BLANK_SHA - @renamed_file = msg.from_path != msg.to_path - @deleted_file = msg.to_id == BLANK_SHA + def init_from_gitaly(diff) + @diff = diff.patch if diff.respond_to?(:patch) + @new_path = encode!(diff.to_path.dup) + @old_path = encode!(diff.from_path.dup) + @a_mode = diff.old_mode.to_s(8) + @b_mode = diff.new_mode.to_s(8) + @new_file = diff.from_id == BLANK_SHA + @renamed_file = diff.from_path != diff.to_path + @deleted_file = diff.to_id == BLANK_SHA end def prune_diff_if_eligible diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index d41256d9a84..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 4491903d788..ba3da781dad 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -26,7 +26,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) - Gitlab::Git::DiffCollection.new(response, options) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb new file mode 100644 index 00000000000..1e117b7e74a --- /dev/null +++ b/lib/gitlab/gitaly_client/diff.rb @@ -0,0 +1,21 @@ +module Gitlab + module GitalyClient + class Diff + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params.each do |key, val| + public_send(:"#{key}=", val) + end + end + + def ==(other) + FIELDS.all? do |field| + public_send(field) == other.public_send(field) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb new file mode 100644 index 00000000000..d84e8d752dc --- /dev/null +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + class DiffStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_diff = nil + + @rpc_response.each do |diff_msg| + if current_diff.nil? + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params[:patch] = diff_msg.raw_patch_data + + current_diff = GitalyClient::Diff.new(diff_params) + else + current_diff.patch += diff_msg.raw_patch_data + end + + if diff_msg.end_of_patch + yield current_diff + current_diff = nil + end + end + end + end + end +end diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 890bd9a3554..b1dbf554e41 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -14,7 +14,7 @@ module Gitlab end def valid? - raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects") end def repos diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1b43440673c..ab38c0c3e34 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -95,7 +95,7 @@ module Gitlab labels = import_issue_labels(raw_issue) assignee_id = nil - if raw_issue.has_key?("owner") + if raw_issue.key?("owner") username = user_map[raw_issue["owner"]["name"]] if username.start_with?("@") @@ -144,7 +144,7 @@ module Gitlab def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift - next if raw_comment.has_key?("deletedBy") + next if raw_comment.key?("deletedBy") content = format_content(raw_comment["content"]) updates = format_updates(raw_comment["updates"]) @@ -235,15 +235,15 @@ module Gitlab def format_updates(raw_updates) updates = [] - if raw_updates.has_key?("status") + if raw_updates.key?("status") updates << "*Status: #{raw_updates["status"]}*" end - if raw_updates.has_key?("owner") + if raw_updates.key?("owner") updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end - if raw_updates.has_key?("cc") + if raw_updates.key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -255,7 +255,7 @@ module Gitlab updates << "*Cc: #{cc.join(", ")}*" end - if raw_updates.has_key?("labels") + if raw_updates.key?("labels") labels = raw_updates["labels"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -267,11 +267,11 @@ module Gitlab updates << "*Labels: #{labels.join(", ")}*" end - if raw_updates.has_key?("mergedInto") + if raw_updates.key?("mergedInto") updates << "*Merged into: ##{raw_updates["mergedInto"]}*" end - if raw_updates.has_key?("blockedOn") + if raw_updates.key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end @@ -279,7 +279,7 @@ module Gitlab updates << "*Blocked on: #{blocked_ons.join(", ")}*" end - if raw_updates.has_key?("blocking") + if raw_updates.key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 36791fae60f..877aa6e6a28 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -25,8 +25,8 @@ module Gitlab def parse_entry(entry) raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) - raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') - raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public') source_pattern = entry['source'] public_path = entry['public'] diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 4c395b4266e..fa182c4deda 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -21,5 +21,13 @@ module Gitlab nil end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2e31f4462f9..85da4c8660b 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -83,7 +83,7 @@ module Gitlab end def valid_level?(level) - options.has_value?(level) + options.value?(level) end def level_name(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 18d8b4f4744..7f27317775c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -129,7 +129,7 @@ module Gitlab 'MaxSessionTime' => terminal[:max_session_time] } } - details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem) details end diff --git a/lib/system_check.rb b/lib/system_check.rb new file mode 100644 index 00000000000..466c39904fa --- /dev/null +++ b/lib/system_check.rb @@ -0,0 +1,21 @@ +# Library to perform System Checks +# +# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck +# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor +# +# This structure decouples checks from Rake tasks and facilitates unit-testing +module SystemCheck + # Executes a bunch of checks for specified component + # + # @param [String] component name of the component relative to the checks being executed + # @param [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + def self.run(component, checks = []) + executor = SimpleExecutor.new(component) + + checks.each do |check| + executor << check + end + + executor.execute + end +end diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb new file mode 100644 index 00000000000..1d72c8d6903 --- /dev/null +++ b/lib/system_check/app/active_users_check.rb @@ -0,0 +1,17 @@ +module SystemCheck + module App + class ActiveUsersCheck < SystemCheck::BaseCheck + set_name 'Active users:' + + def multi_check + active_users = User.active.count + + if active_users > 0 + $stdout.puts active_users.to_s.color(:green) + else + $stdout.puts active_users.to_s.color(:red) + end + end + end + end +end diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb new file mode 100644 index 00000000000..d1fae192350 --- /dev/null +++ b/lib/system_check/app/database_config_exists_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class DatabaseConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Database config exists?' + + def check? + database_config_file = Rails.root.join('config', 'database.yml') + + File.exist?(database_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/database.yml.<your db> to config/database.yml', + 'Check that the information in config/database.yml is correct' + ) + for_more_information( + 'doc/install/databases.md', + 'http://guides.rubyonrails.org/getting_started.html#configuring-a-database' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb new file mode 100644 index 00000000000..198867f7ac6 --- /dev/null +++ b/lib/system_check/app/git_config_check.rb @@ -0,0 +1,42 @@ +module SystemCheck + module App + class GitConfigCheck < SystemCheck::BaseCheck + OPTIONS = { + 'core.autocrlf' => 'input' + }.freeze + + set_name 'Git configured correctly?' + + def check? + correct_options = OPTIONS.map do |name, value| + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + end + + correct_options.all? + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successful (according to their exit code) + # Returns false if any or all subcommands failed. + def repair! + return false unless is_gitlab_user? + + command_success = OPTIONS.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + end + + def show_error + try_fixing_it( + sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"") + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + end + end + end +end diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb new file mode 100644 index 00000000000..c388682dfb4 --- /dev/null +++ b/lib/system_check/app/git_version_check.rb @@ -0,0 +1,29 @@ +module SystemCheck + module App + class GitVersionCheck < SystemCheck::BaseCheck + set_name -> { "Git version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 7, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" + + try_fixing_it( + "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb new file mode 100644 index 00000000000..247aa0994e4 --- /dev/null +++ b/lib/system_check/app/gitlab_config_exists_check.rb @@ -0,0 +1,24 @@ +module SystemCheck + module App + class GitlabConfigExistsCheck < SystemCheck::BaseCheck + set_name 'GitLab config exists?' + + def check? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + + File.exist?(gitlab_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb new file mode 100644 index 00000000000..c609e48e133 --- /dev/null +++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb @@ -0,0 +1,30 @@ +module SystemCheck + module App + class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck + set_name 'GitLab config up to date?' + set_skip_reason "can't check because of previous errors" + + def skip? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + !File.exist?(gitlab_config_file) + end + + def check? + # omniauth or ldap could have been deleted from the file + !Gitlab.config['git_host'] + end + + def show_error + try_fixing_it( + 'Back-up your config/gitlab.yml', + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb new file mode 100644 index 00000000000..d246e058e86 --- /dev/null +++ b/lib/system_check/app/init_script_exists_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class InitScriptExistsCheck < SystemCheck::BaseCheck + set_name 'Init script exists?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def check? + script_path = '/etc/init.d/gitlab' + File.exist?(script_path) + end + + def show_error + try_fixing_it( + 'Install the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb new file mode 100644 index 00000000000..015c7ed1731 --- /dev/null +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module App + class InitScriptUpToDateCheck < SystemCheck::BaseCheck + SCRIPT_PATH = '/etc/init.d/gitlab'.freeze + + set_name 'Init script up-to-date?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def multi_check + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + + unless File.exist?(SCRIPT_PATH) + $stdout.puts "can't check because of previous errors".color(:magenta) + return + end + + recipe_content = File.read(recipe_path) + script_content = File.read(SCRIPT_PATH) + + if recipe_content == script_content + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + + def show_error + try_fixing_it( + 'Re-download the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb new file mode 100644 index 00000000000..3e0c436d6ee --- /dev/null +++ b/lib/system_check/app/log_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class LogWritableCheck < SystemCheck::BaseCheck + set_name 'Log directory writable?' + + def check? + File.writable?(log_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{log_path}", + "sudo chmod -R u+rwX #{log_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def log_path + Rails.root.join('log') + end + end + end +end diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb new file mode 100644 index 00000000000..5eedbacce77 --- /dev/null +++ b/lib/system_check/app/migrations_are_up_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class MigrationsAreUpCheck < SystemCheck::BaseCheck + set_name 'All migrations up?' + + def check? + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) + + migration_status !~ /down\s+\d{14}/ + end + + def show_error + try_fixing_it( + sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb new file mode 100644 index 00000000000..2b46d36fe51 --- /dev/null +++ b/lib/system_check/app/orphaned_group_members_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class OrphanedGroupMembersCheck < SystemCheck::BaseCheck + set_name 'Database contains orphaned GroupMembers?' + set_check_pass 'no' + set_check_fail 'yes' + + def check? + !GroupMember.where('user_id not in (select id from users)').exists? + end + + def show_error + try_fixing_it( + 'You can delete the orphaned records using something along the lines of:', + sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") + ) + end + end + end +end diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb new file mode 100644 index 00000000000..a6ec9f7665c --- /dev/null +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -0,0 +1,37 @@ +module SystemCheck + module App + class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck + set_name 'Projects have namespace:' + set_skip_reason "can't check, you have no projects" + + def skip? + !Project.exists? + end + + def multi_check + $stdout.puts '' + + Project.find_each(batch_size: 100) do |project| + $stdout.print sanitized_message(project) + + if project.namespace + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + end + + def show_error + try_fixing_it( + "Migrate global projects" + ) + for_more_information( + "doc/update/5.4-to-6.0.md in section \"#global-projects\"" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb new file mode 100644 index 00000000000..a0610e73576 --- /dev/null +++ b/lib/system_check/app/redis_version_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class RedisVersionCheck < SystemCheck::BaseCheck + MIN_REDIS_VERSION = '2.8.0'.freeze + set_name "Redis version >= #{MIN_REDIS_VERSION}?" + + def check? + redis_version = run_command(%w(redis-cli --version)) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) + + redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION)) + end + + def show_error + try_fixing_it( + "Update your redis server to a version >= #{MIN_REDIS_VERSION}" + ) + for_more_information( + 'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb new file mode 100644 index 00000000000..fd82f5f8a4a --- /dev/null +++ b/lib/system_check/app/ruby_version_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class RubyVersionCheck < SystemCheck::BaseCheck + set_name -> { "Ruby version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 3, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + try_fixing_it( + "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb new file mode 100644 index 00000000000..99a75e57abf --- /dev/null +++ b/lib/system_check/app/tmp_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class TmpWritableCheck < SystemCheck::BaseCheck + set_name 'Tmp directory writable?' + + def check? + File.writable?(tmp_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{tmp_path}", + "sudo chmod -R u+rwX #{tmp_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def tmp_path + Rails.root.join('tmp') + end + end + end +end diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb new file mode 100644 index 00000000000..7026d0ba075 --- /dev/null +++ b/lib/system_check/app/uploads_directory_exists_check.rb @@ -0,0 +1,21 @@ +module SystemCheck + module App + class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck + set_name 'Uploads directory exists?' + + def check? + File.directory?(Rails.root.join('public/uploads')) + end + + def show_error + try_fixing_it( + "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb new file mode 100644 index 00000000000..7df6c060254 --- /dev/null +++ b/lib/system_check/app/uploads_path_permission_check.rb @@ -0,0 +1,36 @@ +module SystemCheck + module App + class UploadsPathPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory has correct permissions?' + set_skip_reason 'skipped (no uploads folder found)' + + def skip? + !File.directory?(rails_uploads_path) + end + + def check? + File.stat(uploads_fullpath).mode == 040700 + end + + def show_error + try_fixing_it( + "sudo chmod 700 #{uploads_fullpath}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def rails_uploads_path + Rails.root.join('public/uploads') + end + + def uploads_fullpath + File.realpath(rails_uploads_path) + end + end + end +end diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb new file mode 100644 index 00000000000..b276a81eac1 --- /dev/null +++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb @@ -0,0 +1,40 @@ +module SystemCheck + module App + class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory tmp has correct permissions?' + set_skip_reason 'skipped (no tmp uploads folder yet)' + + def skip? + !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp) + end + + def check? + # If tmp upload dir has incorrect permissions, assume others do as well + # Verify drwx------ permissions + File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) + end + + def show_error + try_fixing_it( + "sudo chown -R #{gitlab_user} #{uploads_fullpath}", + "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;", + "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def upload_path_tmp + File.join(uploads_fullpath, 'tmp') + end + + def uploads_fullpath + File.realpath(Rails.root.join('public/uploads')) + end + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb new file mode 100644 index 00000000000..5dcb3f0886b --- /dev/null +++ b/lib/system_check/base_check.rb @@ -0,0 +1,129 @@ +module SystemCheck + # Base class for Checks. You must inherit from here + # and implement the methods below when necessary + class BaseCheck + include ::SystemCheck::Helpers + + # Define a custom term for when check passed + # + # @param [String] term used when check passed (default: 'yes') + def self.set_check_pass(term) + @check_pass = term + end + + # Define a custom term for when check failed + # + # @param [String] term used when check failed (default: 'no') + def self.set_check_fail(term) + @check_fail = term + end + + # Define the name of the SystemCheck that will be displayed during execution + # + # @param [String] name of the check + def self.set_name(name) + @name = name + end + + # Define the reason why we skipped the SystemCheck + # + # This is only used if subclass implements `#skip?` + # + # @param [String] reason to be displayed + def self.set_skip_reason(reason) + @skip_reason = reason + end + + # Term to be displayed when check passed + # + # @return [String] term when check passed ('yes' if not re-defined in a subclass) + def self.check_pass + call_or_return(@check_pass) || 'yes' + end + + ## Term to be displayed when check failed + # + # @return [String] term when check failed ('no' if not re-defined in a subclass) + def self.check_fail + call_or_return(@check_fail) || 'no' + end + + # Name of the SystemCheck defined by the subclass + # + # @return [String] the name + def self.display_name + call_or_return(@name) || self.name + end + + # Skip reason defined by the subclass + # + # @return [String] the reason + def self.skip_reason + call_or_return(@skip_reason) || 'skipped' + end + + # Does the check support automatically repair routine? + # + # @return [Boolean] whether check implemented `#repair!` method or not + def can_repair? + self.class.instance_methods(false).include?(:repair!) + end + + def can_skip? + self.class.instance_methods(false).include?(:skip?) + end + + def is_multi_check? + self.class.instance_methods(false).include?(:multi_check) + end + + # Execute the check routine + # + # This is where you should implement the main logic that will return + # a boolean at the end + # + # You should not print any output to STDOUT here, use the specific methods instead + # + # @return [Boolean] whether check passed or failed + def check? + raise NotImplementedError + end + + # Execute a custom check that cover multiple unities + # + # When using multi_check you have to provide the output yourself + def multi_check + raise NotImplementedError + end + + # Prints troubleshooting instructions + # + # This is where you should print detailed information for any error found during #check? + # + # You may use helper methods to help format the output: + # + # @see #try_fixing_it + # @see #fix_and_rerun + # @see #for_more_infromation + def show_error + raise NotImplementedError + end + + # When implemented by a subclass, will attempt to fix the issue automatically + def repair! + raise NotImplementedError + end + + # When implemented by a subclass, will evaluate whether check should be skipped or not + # + # @return [Boolean] whether or not this check should be skipped + def skip? + raise NotImplementedError + end + + def self.call_or_return(input) + input.respond_to?(:call) ? input.call : input + end + private_class_method :call_or_return + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb new file mode 100644 index 00000000000..c42ae4fe4c4 --- /dev/null +++ b/lib/system_check/helpers.rb @@ -0,0 +1,75 @@ +require 'tasks/gitlab/task_helpers' + +module SystemCheck + module Helpers + include ::Gitlab::TaskHelpers + + # Display a message telling to fix and rerun the checks + def fix_and_rerun + $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) + end + + # Display a formatted list of references (documentation or links) where to find more information + # + # @param [Array<String>] sources one or more references (documentation or links) + def for_more_information(*sources) + $stdout.puts ' For more information see:'.color(:blue) + sources.each do |source| + $stdout.puts " #{source}" + end + end + + def see_installation_guide_section(section) + "doc/install/installation.md in section \"#{section}\"" + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Display a formatted list of instructions on how to fix the issue identified by the #check? + # + # @param [Array<String>] steps one or short sentences with help how to fix the issue + def try_fixing_it(*steps) + steps = steps.shift if steps.first.is_a?(Array) + + $stdout.puts ' Try fixing it:'.color(:blue) + steps.each do |step| + $stdout.puts " #{step}" + end + end + + def sanitized_message(project) + if should_sanitize? + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " + else + "#{project.name_with_namespace.color(:yellow)} ... " + end + end + + def should_sanitize? + if ENV['SANITIZE'] == 'true' + true + else + false + end + end + + def omnibus_gitlab? + Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' + end + + def sudo_gitlab(command) + "sudo -u #{gitlab_user} -H #{command}" + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb new file mode 100644 index 00000000000..dc2d4643a01 --- /dev/null +++ b/lib/system_check/simple_executor.rb @@ -0,0 +1,99 @@ +module SystemCheck + # Simple Executor is current default executor for GitLab + # It is a simple port from display logic in the old check.rake + # + # There is no concurrency level and the output is progressively + # printed into the STDOUT + # + # @attr_reader [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + # @attr_reader [String] component name of the component relative to the checks being executed + class SimpleExecutor + attr_reader :checks + attr_reader :component + + # @param [String] component name of the component relative to the checks being executed + def initialize(component) + raise ArgumentError unless component.is_a? String + + @component = component + @checks = Set.new + end + + # Add a check to be executed + # + # @param [BaseCheck] check class + def <<(check) + raise ArgumentError unless check < BaseCheck + @checks << check + end + + # Executes defined checks in the specified order and outputs confirmation or error information + def execute + start_checking(component) + + @checks.each do |check| + run_check(check) + end + + finished_checking(component) + end + + # Executes a single check + # + # @param [SystemCheck::BaseCheck] check_klass + def run_check(check_klass) + $stdout.print "#{check_klass.display_name} ... " + + check = check_klass.new + + # When implements skip method, we run it first, and if true, skip the check + if check.can_skip? && check.skip? + $stdout.puts check_klass.skip_reason.color(:magenta) + return + end + + # When implements a multi check, we don't control the output + if check.is_multi_check? + check.multi_check + return + end + + if check.check? + $stdout.puts check_klass.check_pass.color(:green) + else + $stdout.puts check_klass.check_fail.color(:red) + + if check.can_repair? + $stdout.print 'Trying to fix error automatically. ...' + if check.repair! + $stdout.puts 'Success'.color(:green) + return + else + $stdout.puts 'Failed'.color(:red) + end + end + + check.show_error + end + end + + private + + # Prints header content for the series of checks to be executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Prints footer content for the series of checks executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + end +end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 0aa21a4bd13..b27f7475115 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -11,4 +11,12 @@ namespace :gettext do "{#{folders}}/**/*.{#{exts}}" ) end + + task :compile do + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 + FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) + + Rake::Task['gettext:pack'].invoke + Rake::Task['gettext:po_to_json'].invoke + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f41c73154f5..63c5e9b9c83 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,5 +1,9 @@ +# Temporary hack, until we migrate all checks to SystemCheck format +require 'system_check' +require 'system_check/helpers' + namespace :gitlab do - desc "GitLab | Check the configuration of GitLab and its environment" + desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check gitlab:incoming_email:check @@ -7,331 +11,38 @@ namespace :gitlab do gitlab:app:check} namespace :app do - desc "GitLab | Check the configuration of the GitLab Rails app" + desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :environment do warn_user_is_not_gitlab - start_checking "GitLab" - - check_git_config - check_database_config_exists - check_migrations_are_up - check_orphaned_group_members - check_gitlab_config_exists - check_gitlab_config_not_outdated - check_log_writable - check_tmp_writable - check_uploads - check_init_script_exists - check_init_script_up_to_date - check_projects_have_namespace - check_redis_version - check_ruby_version - check_git_version - check_active_users - - finished_checking "GitLab" - end - - # Checks - ######################## - - def check_git_config - print "Git configured with autocrlf=input? ... " - - options = { - "core.autocrlf" => "input" - } - - correct_options = options.map do |name, value| - run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value - end - - if correct_options.all? - puts "yes".color(:green) - else - print "Trying to fix Git error automatically. ..." - - if auto_fix_git_config(options) - puts "Success".color(:green) - else - puts "Failed".color(:red) - try_fixing_it( - sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - end - end - end - - def check_database_config_exists - print "Database config exists? ... " - - database_config_file = Rails.root.join("config", "database.yml") - - if File.exist?(database_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/database.yml.<your db> to config/database.yml", - "Check that the information in config/database.yml is correct" - ) - for_more_information( - see_database_guide, - "http://guides.rubyonrails.org/getting_started.html#configuring-a-database" - ) - fix_and_rerun - end - end - - def check_gitlab_config_exists - print "GitLab config exists? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - - if File.exist?(gitlab_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_gitlab_config_not_outdated - print "GitLab config outdated? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exist?(gitlab_config_file) - puts "can't check because of previous errors".color(:magenta) - end - - # omniauth or ldap could have been deleted from the file - unless Gitlab.config['git_host'] - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Backup your config/gitlab.yml", - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_init_script_exists - print "Init script exists? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - script_path = "/etc/init.d/gitlab" - - if File.exist?(script_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Install the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_init_script_up_to_date - print "Init script up-to-date? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") - script_path = "/etc/init.d/gitlab" - - unless File.exist?(script_path) - puts "can't check because of previous errors".color(:magenta) - return - end - - recipe_content = File.read(recipe_path) - script_content = File.read(script_path) - - if recipe_content == script_content - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Redownload the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_migrations_are_up - print "All migrations up? ... " - - migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) - - unless migration_status =~ /down\s+\d{14}/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") - ) - fix_and_rerun - end - end - - def check_orphaned_group_members - print "Database contains orphaned GroupMembers? ... " - if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".color(:red) - try_fixing_it( - "You can delete the orphaned records using something along the lines of:", - sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") - ) - else - puts "no".color(:green) - end - end - - def check_log_writable - print "Log directory writable? ... " - - log_path = Rails.root.join("log") - - if File.writable?(log_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{log_path}", - "sudo chmod -R u+rwX #{log_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - def check_tmp_writable - print "Tmp directory writable? ... " - - tmp_path = Rails.root.join("tmp") - - if File.writable?(tmp_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{tmp_path}", - "sudo chmod -R u+rwX #{tmp_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_uploads - print "Uploads directory setup correctly? ... " - - unless File.directory?(Rails.root.join('public/uploads')) - puts "no".color(:red) - try_fixing_it( - "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - return - end - - upload_path = File.realpath(Rails.root.join('public/uploads')) - upload_path_tmp = File.join(upload_path, 'tmp') - - if File.stat(upload_path).mode == 040700 - unless Dir.exist?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.color(:magenta) - return - end - - # If tmp upload dir has incorrect permissions, assume others do as well - # Verify drwx------ permissions - if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R #{gitlab_user} #{upload_path}", - "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod 700 #{upload_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_redis_version - min_redis_version = "2.8.0" - print "Redis version >= #{min_redis_version}? ... " - - redis_version = run_command(%w(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) - if redis_version && - (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your redis server to a version >= #{min_redis_version}" - ) - for_more_information( - "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq" - ) - fix_and_rerun - end + checks = [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck, + SystemCheck::App::ActiveUsersCheck + ] + + SystemCheck.run('GitLab', checks) end end namespace :gitlab_shell do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -513,33 +224,6 @@ namespace :gitlab do end end - def check_projects_have_namespace - print "projects have namespace: ... " - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - - if project.namespace - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Migrate global projects" - ) - for_more_information( - "doc/update/5.4-to-6.0.md in section \"#global-projects\"" - ) - fix_and_rerun - end - end - end - # Helper methods ######################## @@ -565,6 +249,8 @@ namespace :gitlab do end namespace :sidekiq do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -623,6 +309,8 @@ namespace :gitlab do end namespace :incoming_email do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab @@ -757,6 +445,8 @@ namespace :gitlab do end namespace :ldap do + include SystemCheck::Helpers + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -812,6 +502,8 @@ namespace :gitlab do end namespace :repo do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -826,6 +518,8 @@ namespace :gitlab do end namespace :user do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) @@ -848,55 +542,6 @@ namespace :gitlab do # Helper methods ########################## - def fix_and_rerun - puts " Please fix the error above and rerun the checks.".color(:red) - end - - def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - - puts " For more information see:".color(:blue) - sources.each do |source| - puts " #{source}" - end - end - - def finished_checking(component) - puts "" - puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" - puts "" - end - - def see_database_guide - "doc/install/databases.md" - end - - def see_installation_guide_section(section) - "doc/install/installation.md in section \"#{section}\"" - end - - def sudo_gitlab(command) - "sudo -u #{gitlab_user} -H #{command}" - end - - def gitlab_user - Gitlab.config.gitlab.user - end - - def start_checking(component) - puts "Checking #{component.color(:yellow)} ..." - puts "" - end - - def try_fixing_it(*steps) - steps = steps.shift if steps.first.is_a?(Array) - - puts " Try fixing it:".color(:blue) - steps.each do |step| - puts " #{step}" - end - end - def check_gitlab_shell required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) @@ -909,65 +554,6 @@ namespace :gitlab do end end - def check_ruby_version - required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) - - print "Ruby version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your ruby to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_git_version - required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) - - puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" - print "Git version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your git to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_active_users - puts "Active users: #{User.active.count}" - end - - def omnibus_gitlab? - Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' - end - - def sanitized_message(project) - if should_sanitize? - "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " - else - "#{project.name_with_namespace.color(:yellow)} ... " - end - end - - def should_sanitize? - if ENV['SANITIZE'] == "true" - true - else - false - end - end - def check_repo_integrity(repo_dir) puts "\nChecking repo at #{repo_dir.color(:yellow)}" diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e3c9d3b491c..964aa0fe1bc 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -98,34 +98,30 @@ module Gitlab end end + def gitlab_user + Gitlab.config.gitlab.user + end + + def is_gitlab_user? + return @is_gitlab_user unless @is_gitlab_user.nil? + + current_user = run_command(%w(whoami)).chomp + @is_gitlab_user = current_user == gitlab_user + end + def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user + return if @warned_user_not_gitlab + + unless is_gitlab_user? current_user = run_command(%w(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" - command_success.all? - else - false + @warned_user_not_gitlab = true end end |