diff options
Diffstat (limited to 'lib')
127 files changed, 2471 insertions, 1126 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index ac113c5200d..88f91c07194 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -94,6 +94,8 @@ module API mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments + mount ::API::Events + mount ::API::Features mount ::API::Files mount ::API::Groups mount ::API::Internal @@ -110,6 +112,7 @@ module API mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines + mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 827a38d33da..10f2d5ef6a3 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -68,7 +68,14 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) + pipeline = @project.pipeline_for(ref, commit.sha) + unless pipeline + pipeline = @project.pipelines.create!( + source: :external, + sha: commit.sha, + ref: ref, + user: current_user) + end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( project: @project, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 621b9dcecd9..c6fc17cc391 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -176,7 +176,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 8a54f7f3f05..7cdee8aced7 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -76,6 +76,27 @@ module API end end + desc 'Update an existing deploy key for a project' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + optional :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" + at_least_one_of :title, :can_push + end + put ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys.find(params.delete(:key_id)) + + authorize!(:update_deploy_key, key) + + if key.update_attributes(declared_params(include_missing: false)) + present key, with: Entities::SSHKey + else + render_validation_error!(key) + end + end + desc 'Enable a deploy key for a project' do detail 'This feature was added in GitLab 8.11' success Entities::SSHKey diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 8c5e5c91769..a836df3dc81 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -100,6 +100,8 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :import_status + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } expose :avatar_url do |user, options| user.avatar_url(only_path: false) end @@ -224,7 +226,7 @@ module API end class ProjectSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at @@ -234,7 +236,7 @@ module API end class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at @@ -331,7 +333,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -344,7 +346,7 @@ module API expose :commits, using: Entities::RepoCommit expose :diffs, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -548,7 +550,7 @@ module API end expose :diffs, using: Entities::RepoDiff do |compare, options| - compare.diffs(all_diffs: true).to_a + compare.diffs(limits: false).to_a end expose :compare_timeout do |compare, options| @@ -675,6 +677,7 @@ module API class Variable < Grape::Entity expose :key, :value + expose :protected?, as: :protected end class Pipeline < PipelineBasic @@ -686,6 +689,17 @@ module API expose :coverage end + class PipelineSchedule < Grape::Entity + expose :id + expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active + expose :created_at, :updated_at + expose :owner, using: Entities::UserBasic + end + + class PipelineScheduleDetails < PipelineSchedule + expose :last_pipeline, using: Entities::PipelineBasic + end + class EnvironmentBasic < Grape::Entity expose :id, :name, :slug, :external_url end @@ -742,6 +756,28 @@ module API expose :impersonation end + class FeatureGate < Grape::Entity + expose :key + expose :value + end + + class Feature < Grape::Entity + expose :name + expose :state + expose :gates, using: FeatureGate do |model| + model.gates.map do |gate| + value = model.gate_values[gate.key] + + # By default all gate values are populated. Only show relevant ones. + if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?) + next + end + + { key: gate.key, value: value } + end.compact + end + end + module JobRequest class JobInfo < Grape::Entity expose :name, :stage diff --git a/lib/api/events.rb b/lib/api/events.rb new file mode 100644 index 00000000000..dabdf579119 --- /dev/null +++ b/lib/api/events.rb @@ -0,0 +1,86 @@ +module API + class Events < Grape::API + include PaginationParams + + helpers do + params :event_filter_params do + optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' + optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' + optional :before, type: Date, desc: 'Include only events created before this date' + optional :after, type: Date, desc: 'Include only events created after this date' + end + + params :sort_params do + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return events sorted in ascending and descending order' + end + + def present_events(events) + events = events.reorder(created_at: params[:sort]) + + present paginate(events), with: Entities::Event + end + end + + resource :events do + desc "List currently authenticated user's events" do + detail 'This feature was introduced in GitLab 9.3.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get do + authenticate! + + events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: String, desc: 'The ID or Username of the user' + end + resource :users do + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ':id/events' do + user = find_user(params[:id]) + not_found!('User') unless user + + events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc "List a Project's visible events" do + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ":id/events" do + events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + end +end diff --git a/lib/api/features.rb b/lib/api/features.rb new file mode 100644 index 00000000000..cff0ba2ddff --- /dev/null +++ b/lib/api/features.rb @@ -0,0 +1,36 @@ +module API + class Features < Grape::API + before { authenticated_as_admin! } + + resource :features do + desc 'Get a list of all features' do + success Entities::Feature + end + get do + features = Feature.all + + present features, with: Entities::Feature, current_user: current_user + end + + desc 'Set the gate value for the given feature' do + success Entities::Feature + end + params do + requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' + end + post ':name' do + feature = Feature.get(params[:name]) + + if %w(0 false).include?(params[:value]) + feature.disable + elsif params[:value] == 'true' + feature.enable + else + feature.enable_percentage_of_time(params[:value].to_i) + end + + present feature, with: Entities::Feature, current_user: current_user + end + end + end +end diff --git a/lib/api/files.rb b/lib/api/files.rb index e6ea12c5ab7..25b0968a271 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -10,7 +10,8 @@ module API file_content: attrs[:content], file_content_encoding: attrs[:encoding], author_email: attrs[:author_email], - author_name: attrs[:author_name] + author_name: attrs[:author_name], + last_commit_sha: attrs[:last_commit_id] } end @@ -46,6 +47,7 @@ module API use :simple_file_params requires :content, type: String, desc: 'File content' optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + optional :last_commit_id, type: String, desc: 'Last known commit id for this file' end end @@ -111,7 +113,12 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + + begin + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + rescue ::Files::UpdateService::FileChangedError => e + render_api_error!(e.message, 400) + end if result[:status] == :success status(200) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index ee85b777aff..ebbaed0cbb7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -83,7 +83,7 @@ module API group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute if group.persisted? - present group, with: Entities::Group, current_user: current_user + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to save group #{group.errors.messages}", 400) end @@ -101,8 +101,6 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - at_least_one_of :name, :path, :description, :visibility, - :lfs_enabled, :request_access_enabled end put ':id' do group = find_group!(params[:id]) @@ -151,8 +149,8 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 226a7ddd50e..2c73a6fdc4e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -158,7 +158,7 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) + if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end @@ -256,31 +256,21 @@ module API # project helpers - def filter_projects(projects) - if params[:membership] - projects = projects.merge(current_user.authorized_projects) - end - - if params[:owned] - projects = projects.merge(current_user.owned_projects) - end - - if params[:starred] - projects = projects.merge(current_user.starred_projects) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects = projects.where(archived: params[:archived]) + def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end + def project_finder_params + finder_params = {} + finder_params[:owned] = true if params[:owned].present? + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = params[:archived] + finder_params[:search] = params[:search] if params[:search] + finder_params + end + # file helpers def uploaded_file(field, uploads_path) @@ -321,6 +311,16 @@ module API end end + def present_artifacts!(artifacts_file) + return not_found! unless artifacts_file.exists? + + if artifacts_file.file_storage? + present_file!(artifacts_file.path, artifacts_file.filename) + else + redirect_to(artifacts_file.url) + end + end + private def private_token diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 264df7271a3..d3732d67622 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,22 @@ module API @project, @wiki = Gitlab::RepoPath.parse(params[:project]) end end + + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + def gl_repository + Gitlab::GlRepository.gl_repository(project, wiki?) + end + + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + def repository_path + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9ebd4841296..38631953014 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -32,31 +32,23 @@ module API actor.update_last_used_at if actor.is_a?(Key) - access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_status = access_checker + access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_checker = access_checker_klass .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) - .check(params[:action], params[:changes]) - response = { status: access_status.status, message: access_status.message } - - if access_status.status - log_user_activity(actor) - - # Project id to pass between components that don't share/don't have - # access to the same filesystem mounts - response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) - - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - response[:repository_path] = - if wiki? - project.wiki.repository.path_to_repo - else - project.repository.path_to_repo - end + begin + access_checker.check(params[:action], params[:changes]) + rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e + return { status: false, message: e.message } end - response + log_user_activity(actor) + + { + status: true, + gl_repository: gl_repository, + repository_path: repository_path + } end post "/lfs_authenticate" do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 0223957fde1..8a67de10bca 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -224,16 +224,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb new file mode 100644 index 00000000000..93d89209934 --- /dev/null +++ b/lib/api/pipeline_schedules.rb @@ -0,0 +1,131 @@ +module API + class PipelineSchedules < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all pipeline schedules' do + success Entities::PipelineSchedule + end + params do + use :pagination + optional :scope, type: String, values: %w[active inactive], + desc: 'The scope of pipeline schedules' + end + get ':id/pipeline_schedules' do + authorize! :read_pipeline_schedule, user_project + + schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) + .preload([:owner, :last_pipeline]) + present paginate(schedules), with: Entities::PipelineSchedule + end + + desc 'Get a single pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + get ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + present pipeline_schedule, with: Entities::PipelineScheduleDetails + end + + desc 'Create a new pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :description, type: String, desc: 'The description of pipeline schedule' + requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' + optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' + end + post ':id/pipeline_schedules' do + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = Ci::CreatePipelineScheduleService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if pipeline_schedule.persisted? + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Edit a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + optional :description, type: String, desc: 'The description of pipeline schedule' + optional :ref, type: String, desc: 'The branch/tag name will be triggered' + optional :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, desc: 'The timezone' + optional :active, type: Boolean, desc: 'The activation of pipeline schedule' + end + put ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.update(declared_params(include_missing: false)) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Take ownership of a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.own!(current_user) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Delete a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :admin_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + status :accepted + present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + end + end + + helpers do + def pipeline_schedule + @pipeline_schedule ||= + user_project.pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)) + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 9117704aa46..e505cae3992 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -47,7 +47,7 @@ module API new_pipeline = Ci::CreatePipelineService.new(user_project, current_user, declared_params(include_missing: false)) - .execute(ignore_skip_ci: true, save_on_errors: false) + .execute(:api, ignore_skip_ci: true, save_on_errors: false) if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 98bc9c28527..64efe82a937 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -49,6 +49,7 @@ module API requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' requires :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' requires :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' @@ -77,6 +78,7 @@ module API optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' optional :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ed5004e8d1a..56046742e08 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -21,6 +21,7 @@ module API optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' end params :optional_params do @@ -58,6 +59,8 @@ module API optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' end params :create_params do @@ -65,16 +68,19 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(projects, options = {}) + def present_projects(options = {}) + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + options = options.reverse_merge( - with: Entities::Project, - current_user: current_user, - simple: params[:simple] + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user ) - - projects = filter_projects(projects) - projects = projects.with_statistics if options[:statistics] - options[:with] = Entities::BasicProjectDetails if options[:simple] + options[:with] = Entities::BasicProjectDetails if params[:simple] present paginate(projects), options end @@ -88,8 +94,7 @@ module API use :statistics_params end get do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] + present_projects end desc 'Create new project' do @@ -104,7 +109,7 @@ module API end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -124,6 +129,7 @@ module API params do requires :name, type: String, desc: 'The name of the project' requires :user_id, type: Integer, desc: 'The ID of a user' + optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' use :optional_params use :create_params @@ -161,16 +167,6 @@ module API user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end - desc 'Get events for a single project' do - success Entities::Event - end - params do - use :pagination - end - get ":id/events" do - present paginate(user_project.events.recent), with: Entities::Event - end - desc 'Fork new project for the current user or provided namespace.' do success Entities::Project end @@ -225,6 +221,7 @@ module API :request_access_enabled, :shared_runners_enabled, :snippets_enabled, + :tag_list, :visibility, :wiki_enabled ] @@ -241,7 +238,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 6fbb02cb3aa..4552115b3e2 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -141,7 +141,7 @@ module API patch '/:id/trace' do job = authenticate_job! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -241,16 +241,7 @@ module API get '/:id/artifacts' do job = authenticate_job! - artifacts_file = job.artifacts_file - unless artifacts_file.file_storage? - return redirect_to job.artifacts_file.url - end - - unless artifacts_file.exists? - not_found! - end - - present_file!(artifacts_file.path, artifacts_file.filename) + present_artifacts!(job.artifacts_file) end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 82f513c984e..25027c3b114 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -110,6 +110,7 @@ module API optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 53f5953a8fb..c630c24c339 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -58,6 +58,7 @@ module API requires :title, type: String, desc: 'The title of a snippet' requires :file_name, type: String, desc: 'The name of a snippet file' requires :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, default: 'internal', @@ -85,6 +86,7 @@ module API optional :title, type: String, desc: 'The title of a snippet' optional :file_name, type: String, desc: 'The name of a snippet file' optional :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 05b4b490e27..df4632346dd 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,7 +5,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' + declared_params.key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/users.rb b/lib/api/users.rb index 3d83720b7b9..dda64715ee1 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -124,10 +124,6 @@ module API optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes - at_least_one_of :email, :password, :name, :username, :skype, :linkedin, - :twitter, :website_url, :organization, :projects_limit, - :extern_uid, :provider, :bio, :location, :admin, - :can_create_group, :confirm, :external end put ":id" do authenticated_as_admin! @@ -286,13 +282,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserWorker.perform_async(current_user.id, user.id) + user.delete_async(deleted_by: current_user, params: params) end desc 'Block a user. Available only for admins.' @@ -327,27 +324,6 @@ module API end end - desc 'Get the contribution events of a specified user' do - detail 'This feature was introduced in GitLab 8.13.' - success Entities::Event - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/events' do - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - events = user.events. - merge(ProjectsFinder.new(current_user: current_user).execute). - references(:project). - with_associations. - recent - - present paginate(events), with: Entities::Event - end - params do requires :user_id, type: Integer, desc: 'The ID of the user' end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 21935922414..93ad9eb26b8 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -225,16 +225,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 674de592f0a..5936f4700aa 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -167,7 +167,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb index bbb174b6003..b90e2061da3 100644 --- a/lib/api/v3/deploy_keys.rb +++ b/lib/api/v3/deploy_keys.rb @@ -41,6 +41,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/#{path}" do params[:key].strip! diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 2e1b243c2db..7c5065dee90 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -226,7 +226,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index 0f234d4cdad..d9e76560d03 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -14,6 +14,33 @@ module API authorize! access_level, merge_request merge_request end + + # project helpers + + def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) + end + + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) + end end end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 164612cb8dd..20976b9dd08 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -44,7 +44,7 @@ module API end def set_only_allow_merge_if_pipeline_succeeds! - if params.has_key?(:only_allow_merge_if_build_succeeds) + if params.key?(:only_allow_merge_if_build_succeeds) params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) end end @@ -147,7 +147,7 @@ module API get '/starred' do authenticate! - present_projects current_user.viewable_starred_projects + present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute end desc 'Get all projects for admin user' do diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb index 81ae4e8137d..d5b90e435ba 100644 --- a/lib/api/v3/time_tracking_endpoints.rb +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -6,7 +6,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.key?(:issue_id) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 5acde41551b..381c4ef50b0 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -42,6 +42,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) @@ -59,13 +60,14 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - if variable.update(value: params[:value]) + if variable.update(declared_params(include_missing: false).except(:key)) present variable, with: Entities::Variable else render_validation_error!(variable) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 51fa3867e67..1f4bda6f588 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.artifacts_path) + super('artifacts', ArtifactUploader.local_artifacts_store) end def create_files_dir diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6b29600a751..a1685c77916 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,15 +7,15 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - if project.empty_repo? - $progress.puts "[SKIPPED]".color(:cyan) + if empty_repo?(project) + progress.puts "[SKIPPED]".color(:cyan) else in_path(path_to_project_repo) do |dir| FileUtils.mkdir_p(path_to_tars(project)) @@ -23,10 +23,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -34,12 +31,9 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -48,19 +42,16 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - $progress.print " * #{wiki.path_with_namespace} ... " - if wiki.repository.empty? - $progress.puts " [SKIPPED]".color(:cyan) + progress.print " * #{wiki.path_with_namespace} ... " + if empty_repo?(wiki) + progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(wiki, cmd.join(' '), output) end end end @@ -80,7 +71,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -94,12 +85,9 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end in_path(path_to_tars(project)) do |dir| @@ -107,10 +95,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end @@ -119,7 +104,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - $progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -129,22 +114,19 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end end - $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) + progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" @@ -201,8 +183,25 @@ module Backup private + def progress_warn(project, cmd, output) + progress.puts "[WARNING] Executing #{cmd}".color(:orange) + progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + end + + def empty_repo?(project_or_wiki) + project_or_wiki.repository.empty_repo? + rescue => e + progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange) + + false + end + def repository_storage_paths_args Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end + + def progress + $progress + end end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index c2503fa2adc..d99a3bfa625 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -163,14 +163,15 @@ module Banzai # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) if RequestStore.active? + ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] - to_query = ids.map(&:to_i) - cache.keys + to_query = ids - cache.keys unless to_query.empty? collection.where(id: to_query).each { |row| cache[row.id] = row } end - cache.values + cache.values_at(*ids) else collection.where(id: ids) end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 4f8efe03bae..c52acbc3ddc 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -22,11 +22,11 @@ module Bitbucket end def inline? - raw.has_key?('inline') + raw.key?('inline') end def has_parent? - raw.has_key?('parent') + raw.key?('parent') end private diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 67b269b330c..e2e91ce99cd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -88,7 +88,7 @@ module Ci patch ":id/trace.txt" do build = authenticate_build! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -187,14 +187,14 @@ module Ci build = authenticate_build! artifacts_file = build.artifacts_file - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - unless artifacts_file.exists? not_found! end + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + present_file!(artifacts_file.path, artifacts_file.filename) end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b06474cda7f..22af2671b18 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -50,10 +50,23 @@ module Ci end end + def stage_seeds(pipeline) + trigger_request = pipeline.trigger_requests.first + + seeds = @stages.uniq.map do |stage| + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, trigger_request) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + def build_attributes(name) job = @jobs[name.to_sym] || {} - { - stage_idx: @stages.index(job[:stage]), + + { stage_idx: @stages.index(job[:stage]), stage: job[:stage], commands: job[:commands], tag_list: job[:tags] || [], @@ -71,8 +84,7 @@ module Ci dependencies: job[:dependencies], after_script: job[:after_script], environment: job[:environment] - }.compact - } + }.compact } end def self.validation_message(content) diff --git a/lib/feature.rb b/lib/feature.rb new file mode 100644 index 00000000000..5650a1c1334 --- /dev/null +++ b/lib/feature.rb @@ -0,0 +1,53 @@ +require 'flipper/adapters/active_record' + +class Feature + # Classes to override flipper table names + class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature + # Using `self.table_name` won't work. ActiveRecord bug? + superclass.table_name = 'features' + end + + class FlipperGate < Flipper::Adapters::ActiveRecord::Gate + superclass.table_name = 'feature_gates' + end + + class << self + def all + flipper.features.to_a + end + + def get(key) + flipper.feature(key) + end + + def persisted?(feature) + # Flipper creates on-memory features when asked for a not-yet-created one. + # If we want to check if a feature has been actually set, we look for it + # on the persisted features list. + all.map(&:name).include?(feature.name) + end + + def enabled?(key) + get(key).enabled? + end + + def enable(key) + get(key).enable + end + + def disable(key) + get(key).disable + end + + private + + def flipper + @flipper ||= begin + adapter = Flipper::Adapters::ActiveRecord.new( + feature_class: FlipperFeature, gate_class: FlipperGate) + + Flipper.new(adapter) + end + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 099c45dcfb7..da07ba2f2a3 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,8 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) + REGISTRY_SCOPES = [:read_registry].freeze + # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze @@ -11,8 +13,10 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze + AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze + # Other available scopes - OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze + OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) @@ -26,8 +30,8 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password) || oauth_access_token_check(login, password) || - user_with_password_for_git(login, password) || personal_access_token_check(password) || + user_with_password_for_git(login, password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) @@ -109,6 +113,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) @@ -121,17 +126,23 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_api_token?(token) - Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_api_token?(token) + token && token.accessible? && valid_scoped_token?(token, ["api"]) end - def valid_api_token?(token) - AccessTokenValidationService.new(token).include_any_scope?(['api']) + def valid_scoped_token?(token, scopes) + AccessTokenValidationService.new(token).include_any_scope?(scopes) + end + + def abilities_for_scope(scopes) + scopes.map do |scope| + self.public_send(:"#{scope}_scope_authentication_abilities") + end.flatten.uniq end def lfs_token_check(login, password) @@ -202,6 +213,16 @@ module Gitlab :create_container_image ] end + alias_method :api_scope_authentication_abilities, :full_authentication_abilities + + def read_registry_scope_authentication_abilities + [:read_container_image] + end + + # The currently used auth method doesn't allow any actions for this scope + def read_user_scope_authentication_abilities + [] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 39b86c61a18..75451cf8aa9 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -15,6 +15,10 @@ module Gitlab def success? actor.present? || type == :ci end + + def failed? + !success? + end end end end diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb index 2700a5a2ad5..05994bee79d 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -45,9 +45,9 @@ module Gitlab end def format_response(response) - response[:text] = format(response[:text]) if response.has_key?(:text) + response[:text] = format(response[:text]) if response.key?(:text) - if response.has_key?(:attachments) + if response.key?(:attachments) response[:attachments].each do |attachment| attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] attachment[:text] = format(attachment[:text]) if attachment[:text] diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index c984eb20606..b6805230348 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,6 +1,20 @@ module Gitlab module Checks class ChangeAccess + ERROR_MESSAGES = { + push_code: 'You are not allowed to push code to this project.', + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.', + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( @@ -17,22 +31,20 @@ module Gitlab end def exec - return GitAccessStatus.new(true) if skip_authorization + return true if skip_authorization - error = push_checks || branch_checks || tag_checks + push_checks + branch_checks + tag_checks - if error - GitAccessStatus.new(false, error) - else - GitAccessStatus.new(true) - end + true end protected def push_checks if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] end end @@ -40,7 +52,7 @@ module Gitlab return unless @branch_name if deletion? && @branch_name == project.default_branch - return "The default branch of a project cannot be deleted." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] end protected_branch_checks @@ -50,7 +62,7 @@ module Gitlab return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? - return "You are not allowed to force push code to a protected branch on this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] end if deletion? @@ -62,22 +74,22 @@ module Gitlab def protected_branch_deletion_checks unless user_access.can_delete_branch?(@branch_name) - return 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.' + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] end unless protocol == 'web' - 'You can only delete protected branches using the web interface.' + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] end end def protected_branch_push_checks if matching_merge_request? unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - "You are not allowed to merge code into protected branches on this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] end else unless user_access.can_push_to_branch?(@branch_name) - "You are not allowed to push code to protected branches on this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] end end end @@ -86,7 +98,7 @@ module Gitlab return unless @tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) - return "You are not allowed to change existing tags on this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] end protected_tag_checks @@ -95,11 +107,11 @@ module Gitlab def protected_tag_checks return unless ProtectedTag.protected?(project, @tag_name) - return "Protected tags cannot be updated." if update? - return "Protected tags cannot be deleted." if deletion? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? unless user_access.can_create_tag?(@tag_name) - return "You are not allowed to create this tag as it is protected." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb new file mode 100644 index 00000000000..f81f9347b4d --- /dev/null +++ b/lib/gitlab/ci/stage/seed.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + module Stage + class Seed + attr_reader :pipeline + delegate :project, to: :pipeline + + def initialize(pipeline, stage, jobs) + @pipeline = pipeline + @stage = { name: stage } + @jobs = jobs.to_a.dup + end + + def user=(current_user) + @jobs.map! do |attributes| + attributes.merge(user: current_user) + end + end + + def stage + @stage.merge(project: project) + end + + def builds + trigger = pipeline.trigger_requests.first + + @jobs.map do |attributes| + attributes.merge(project: project, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: trigger) + end + end + + def create! + pipeline.stages.create!(stage).tap do |stage| + builds_attributes = builds.map do |attributes| + attributes.merge(stage_id: stage.id) + end + + pipeline.builds.create!(builds_attributes).each do |build| + yield build if block_given? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 57b533bad99..439ef0ce015 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,7 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_build_path(subject.project.namespace, + cancel_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index 3fec2c5d4db..b173c23fba4 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,7 +8,7 @@ module Gitlab end def details_path - namespace_project_build_path(subject.project.namespace, + namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c6139f1b716..e80f3263794 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 505f80848b2..56303e4cb17 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,7 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_build_path(subject.project.namespace, + retry_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 0b5199e5483..2778d6f3b52 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb new file mode 100644 index 00000000000..def1373d8cf --- /dev/null +++ b/lib/gitlab/ci_access.rb @@ -0,0 +1,9 @@ +module Gitlab + # For backwards compatibility, generic CI (which is a build without a user) is + # allowed to :build_download_code without any other checks. + class CiAccess + def can_do_action?(action) + action == :build_download_code + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 15992b77680..060e013183f 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -28,7 +28,7 @@ module Gitlab union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) - @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fe..48735fd197d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -8,45 +8,62 @@ module Gitlab end end - def ensure_application_settings! - return fake_application_settings unless connect_to_db? + delegate :sidekiq_throttling_enabled?, to: :current_application_settings - unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - begin - settings = ::ApplicationSetting.current - # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Redis::BaseError, ::Errno::ENOENT - settings = ::ApplicationSetting.last - end + def fake_application_settings + OpenStruct.new(::ApplicationSetting.defaults) + end - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + private + + def ensure_application_settings! + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + settings = retrieve_settings_from_database? end settings || in_memory_application_settings end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def retrieve_settings_from_database? + settings = retrieve_settings_from_database_cache? + return settings if settings.present? + + return fake_application_settings unless connect_to_db? + + begin + db_settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + db_settings = ::ApplicationSetting.last + end + db_settings || ::ApplicationSetting.create_from_defaults + end + + def retrieve_settings_from_database_cache? + begin + settings = ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT + # In case Redis isn't running or the Redis UNIX socket file is not available + settings = nil + end + settings + end def in_memory_application_settings @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct fake_application_settings end - def fake_application_settings - OpenStruct.new(::ApplicationSetting.defaults) - end - - private - def connect_to_db? # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + ActiveRecord::Base.connection.table_exists?('application_settings') && + !ActiveRecord::Migrator.needs_migration? rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 182a30fd74d..e47fb85b5ee 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages_name, + stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 7948782aecc..371cbe04b9b 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -37,6 +37,16 @@ module Gitlab def complete? start_sha && head_sha end + + def compare_in(project) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + project.commit(head_sha) + else + straight = start_sha == base_sha + CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + end + end end end end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 79836a2fbab..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -7,7 +7,7 @@ module Gitlab delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0a15c6d9358..bd52ae47e9f 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -59,6 +59,10 @@ module Gitlab type == 'match' end + def discussable? + !['match', 'new-nonewline', 'old-nonewline'].include?(type) + end + def as_json(opts = nil) { type: type, diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 4d96778a2b2..f80afb20f0c 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -145,23 +145,9 @@ module Gitlab private def find_diff_file(repository) - # We're at the initial commit, so just get that as we can't compare to anything. - compare = - if Gitlab::Git.blank_ref?(start_sha) - Gitlab::Git::Commit.find(repository.raw_repository, head_sha) - else - Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ) - end - - diff = compare.diffs(paths: paths).first - - return unless diff + return unless diff_refs.complete? - Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index dcabb5f7fe5..b68a1636814 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -216,7 +216,7 @@ module Gitlab def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths) + compare.diffs(paths: paths, expanded: true) end def position(diff_file, old_line, new_line) diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 6c69cd9e6a9..ea035e33eff 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -42,7 +42,7 @@ module Gitlab return unless compare # This diff is more moderated in number of files and lines - @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files + @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files end def diffs_count diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..781f9c56a42 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect && detect[:encoding] + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 270d67dd50c..7f884183bb1 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -6,12 +6,13 @@ module Gitlab end def call(env) - route = Gitlab::EtagCaching::Router.match(env) + request = Rack::Request.new(env) + route = Gitlab::EtagCaching::Router.match(request) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(env) + etag, cached_value_present = get_etag(request) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -27,8 +28,8 @@ module Gitlab private - def get_etag(env) - cache_key = env['PATH_INFO'] + def get_etag(request) + cache_key = request.path store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index cc285162b44..dccc66b3918 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -9,9 +9,11 @@ module Gitlab # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route # - Ending in `issues/id`/realtime_changes` for the `issue_title` route USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests new].freeze + commit pipelines merge_requests builds + new environments].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES - RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) + ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), @@ -40,11 +42,19 @@ module Gitlab Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z), + 'project_build' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), + 'environments' ) ].freeze - def self.match(env) - ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + def self.match(request) + ROUTES.find { |route| route.regexp.match(request.path_info) } end end end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index c1b31618e0d..d60e607b02b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection @@ -88,6 +88,7 @@ module Gitlab new( id: blob_entry[:oid], name: blob_entry[:name], + size: 0, data: '', path: path, commit_id: sha diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 297531db4cc..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 696a2acd5e3..78e440395a5 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -3,7 +3,7 @@ module Gitlab class Compare attr_reader :head, :base, :straight - def initialize(repository, base, head, straight = false) + def initialize(repository, base, head, straight: false) @repository = repository @straight = straight diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index deade337354..8926aa19925 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff @@ -15,15 +15,30 @@ module Gitlab alias_method :deleted_file?, :deleted_file alias_method :renamed_file?, :renamed_file + attr_accessor :expanded + + # We need this accessor because of `to_hash` and `init_from_hash` attr_accessor :too_large - # The maximum size of a diff to display. - DIFF_SIZE_LIMIT = 102400 # 100 KB + class << self + # The maximum size of a diff to display. + def size_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 200.kilobytes + else + 100.kilobytes + end + end - # The maximum size before a diff is collapsed. - DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + # The maximum size before a diff is collapsed. + def collapse_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 100.kilobytes + else + 10.kilobytes + end + end - class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -152,7 +167,7 @@ module Gitlab :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, :ignore_filemode, :recurse_ignored_dirs, :paths, - :max_files, :max_lines, :all_diffs, :no_collapse] + :max_files, :max_lines, :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -177,16 +192,18 @@ module Gitlab end end - def initialize(raw_diff, collapse: false) + def initialize(raw_diff, expanded: true) + @expanded = expanded + case raw_diff when Hash init_from_hash(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff, collapse: collapse) - when Gitaly::CommitDiffResponse + init_from_rugged(raw_diff) + when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Gitaly::CommitDelta init_from_gitaly(raw_diff) when nil @@ -226,17 +243,13 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.size_limit else @too_large end end - def collapsible? - @diff.bytesize >= DIFF_COLLAPSE_LIMIT - end - - def prune_large_diff! + def too_large! @diff = '' @line_count = 0 @too_large = true @@ -244,10 +257,11 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - false + + @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit end - def prune_collapsed_diff! + def collapse! @diff = '' @line_count = 0 @collapsed = true @@ -255,9 +269,9 @@ module Gitlab private - def init_from_rugged(rugged, collapse: false) + def init_from_rugged(rugged) if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged, collapse: collapse) + init_from_rugged_patch(rugged) d = rugged.delta else d = rugged @@ -272,10 +286,10 @@ module Gitlab @deleted_file = d.deleted? end - def init_from_rugged_patch(patch, collapse: false) + def init_from_rugged_patch(patch) # Don't bother initializing diffs that are too large. If a diff is # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch, collapse) + return if !patch.delta.binary? && prune_large_patch(patch) @diff = encode!(strip_diff_headers(patch.to_s)) end @@ -288,40 +302,43 @@ module Gitlab end end - def init_from_gitaly(msg) - @diff = msg.raw_chunks.join if msg.respond_to?(:raw_chunks) - @new_path = encode!(msg.to_path.dup) - @old_path = encode!(msg.from_path.dup) - @a_mode = msg.old_mode.to_s(8) - @b_mode = msg.new_mode.to_s(8) - @new_file = msg.from_id == BLANK_SHA - @renamed_file = msg.from_path != msg.to_path - @deleted_file = msg.to_id == BLANK_SHA + def init_from_gitaly(diff) + @diff = diff.patch if diff.respond_to?(:patch) + @new_path = encode!(diff.to_path.dup) + @old_path = encode!(diff.from_path.dup) + @a_mode = diff.old_mode.to_s(8) + @b_mode = diff.new_mode.to_s(8) + @new_file = diff.from_id == BLANK_SHA + @renamed_file = diff.from_path != diff.to_path + @deleted_file = diff.to_id == BLANK_SHA end - def prune_diff_if_eligible(collapse = false) - prune_large_diff! if too_large? - prune_collapsed_diff! if collapse && collapsible? + def prune_diff_if_eligible + if too_large? + too_large! + elsif collapsed? + collapse! + end end # If the patch surpasses any of the diff limits it calls the appropiate # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch, collapse) + def prune_large_patch(patch) size = 0 patch.each_hunk do |hunk| hunk.each_line do |line| size += line.content.bytesize - if size >= DIFF_SIZE_LIMIT - prune_large_diff! + if size >= self.class.size_limit + too_large! return true end end end - if collapse && size >= DIFF_COLLAPSE_LIMIT - prune_collapsed_diff! + if !expanded && size >= self.class.collapse_limit + collapse! return true end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 898a5ae15f2..334e06a6eca 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,12 +9,12 @@ module Gitlab @iterator = iterator @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5120 # Average 5 KB per file + @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file - @all_diffs = !!options.fetch(:all_diffs, false) - @no_collapse = !!options.fetch(:no_collapse, true) + @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @enforce_limits = !!options.fetch(:limits, true) + @expanded = !!options.fetch(:expanded, true) @line_count = 0 @byte_count = 0 @@ -88,23 +88,23 @@ module Gitlab @iterator.each do |raw| @empty = false - if !@all_diffs && i >= @max_files + if @enforce_limits && i >= @max_files @overflow = true break end - collapse = !@all_diffs && !@no_collapse + expanded = !@enforce_limits || @expanded - diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if collapse && over_safe_limits?(i) - diff.prune_collapsed_diff! + if !expanded && over_safe_limits?(i) + diff.collapse! end @line_count += diff.line_count @byte_count += diff.diff.bytesize - if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index b9f1ac144b6..9d6adbdb4ac 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1006,31 +1006,39 @@ module Gitlab # Parses the contents of a .gitmodules file and returns a hash of # submodule information. def parse_gitmodules(commit, content) - results = {} + modules = {} - current = "" - content.split("\n").each do |txt| - if txt =~ /^\s*\[/ - current = txt.match(/(?<=").*(?=")/)[0] - results[current] = {} - else - next unless results[current] - match_data = txt.match(/(\w+)\s*=\s*(.*)/) - next unless match_data - target = match_data[2].chomp - results[current][match_data[1]] = target + name = nil + content.each_line do |line| + case line.strip + when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header + name = $~[:name] + modules[name] = {} + when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair + key = $~[:key] + value = $~[:value].chomp + + next unless name && modules[name] + + modules[name][key] = value - if match_data[1] == "path" + if key == 'path' begin - results[current]["id"] = blob_content(commit, target) + modules[name]['id'] = blob_content(commit, value) rescue InvalidBlobName - results.delete(current) + # The current entry is invalid + modules.delete(name) + name = nil end end + when /\A#/ # Comment + next + else # Invalid line + name = nil end end - results + modules end # Returns true if +commit+ introduced changes to +path+, using commit @@ -1086,7 +1094,12 @@ module Gitlab elsif tmp_entry.nil? return nil else - tmp_entry = rugged.lookup(tmp_entry[:oid]) + begin + tmp_entry = rugged.lookup(tmp_entry[:oid]) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + return nil + end + return nil unless tmp_entry.type == :tree tmp_entry = tmp_entry[dir] end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index d41256d9a84..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 99724db8da2..0a19d24eb20 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -3,33 +3,39 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) + NotFoundError = Class.new(StandardError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', deploy_key_upload: 'This deploy key does not have write access to this project.', - no_repo: 'A repository for this project does not exist yet.' + no_repo: 'A repository for this project does not exist yet.', + project_not_found: 'The project you were looking for could not be found.', + account_blocked: 'Your account has been blocked.', + command_not_allowed: "The command you're trying to execute is not allowed.", + upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities + attr_reader :actor, :project, :protocol, :authentication_abilities def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities - @user_access = UserAccess.new(user, project: project) end def check(cmd, changes) check_protocol! check_active_user! check_project_accessibility! + check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! @@ -40,9 +46,7 @@ module Gitlab check_push_access!(changes) end - build_status_object(true) - rescue UnauthorizedError => ex - build_status_object(false, ex.message) + true end def guest_can_download_code? @@ -73,19 +77,39 @@ module Gitlab return if deploy_key? if user && !user_access.allowed? - raise UnauthorizedError, "Your account has been blocked." + raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] end end def check_project_accessibility! if project.blank? || !can_read_project? - raise UnauthorizedError, 'The project you were looking for could not be found.' + raise NotFoundError, ERROR_MESSAGES[:project_not_found] + end + end + + def check_command_disabled!(cmd) + if upload_pack?(cmd) + check_upload_pack_disabled! + elsif receive_pack?(cmd) + check_receive_pack_disabled! + end + end + + def check_upload_pack_disabled! + if http? && upload_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + end + end + + def check_receive_pack_disabled! + if http? && receive_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, "The command you're trying to execute is not allowed." + raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] end end @@ -138,11 +162,9 @@ module Gitlab # Iterate over all changes to find if user allowed all of them to be applied changes_list.each do |change| - status = check_single_change_access(change) - unless status.allowed? - # If user does not have access to make at least one change - cancel all push - raise UnauthorizedError, status.message - end + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change) end end @@ -168,14 +190,40 @@ module Gitlab actor.is_a?(DeployKey) end + def ci? + actor == :ci + end + def can_read_project? - if deploy_key + if deploy_key? deploy_key.has_access_to?(project) elsif user user.can?(:read_project, project) + elsif ci? + true # allow CI (build without a user) for backwards compatibility end || Guest.can?(:read_project, project) end + def http? + protocol == 'http' + end + + def upload_pack?(command) + command == 'git-upload-pack' + end + + def receive_pack?(command) + command == 'git-receive-pack' + end + + def upload_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.upload_pack + end + + def receive_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.receive_pack + end + protected def user @@ -185,15 +233,19 @@ module Gitlab case actor when User actor - when DeployKey - nil when Key - actor.user + actor.user unless actor.is_a?(DeployKey) + when :ci + nil end end - def build_status_object(status, message = '') - Gitlab::GitAccessStatus.new(status, message) + def user_access + @user_access ||= if ci? + CiAccess.new + else + UserAccess.new(user, project: project) + end end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb deleted file mode 100644 index 09bb01be694..00000000000 --- a/lib/gitlab/git_access_status.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Gitlab - class GitAccessStatus - attr_accessor :status, :message - alias_method :allowed?, :status - - def initialize(status, message = '') - @status = status - @message = message - end - - def to_json(opts = nil) - { status: @status, message: @message }.to_json(opts) - end - end -end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 67eaa5e088d..1fe5155c093 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,5 +1,9 @@ module Gitlab class GitAccessWiki < GitAccess + ERROR_MESSAGES = { + write_to_wiki: "You are not allowed to write to this project's wiki." + }.freeze + def guest_can_download_code? Guest.can?(:download_wiki_code, project) end @@ -9,11 +13,11 @@ module Gitlab end def check_single_change_access(change) - if user_access.can_do_action?(:create_wiki) - build_status_object(true) - else - build_status_object(false, "You are not allowed to write to this project's wiki.") + unless user_access.can_do_action?(:create_wiki) + raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + + true end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 72466700c05..2343446bf22 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -2,6 +2,12 @@ require 'gitaly' module Gitlab module GitalyClient + module MigrationStatus + DISABLED = 1 + OPT_IN = 2 + OPT_OUT = 3 + end + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MUTEX = Mutex.new @@ -46,8 +52,20 @@ module Gitlab Gitlab.config.gitaly.enabled end - def self.feature_enabled?(feature) - enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + return false if !enabled? || status == MigrationStatus::DISABLED + + feature = Feature.get("gitaly_#{feature}") + + # If the feature hasn't been set, turn it on if it's opt-out + return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + feature.enabled? end def self.migrate(feature) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 4491903d788..ba3da781dad 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -26,7 +26,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) - Gitlab::Git::DiffCollection.new(response, options) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) end def commit_deltas(commit) diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb new file mode 100644 index 00000000000..1e117b7e74a --- /dev/null +++ b/lib/gitlab/gitaly_client/diff.rb @@ -0,0 +1,21 @@ +module Gitlab + module GitalyClient + class Diff + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params.each do |key, val| + public_send(:"#{key}=", val) + end + end + + def ==(other) + FIELDS.all? do |field| + public_send(field) == other.public_send(field) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb new file mode 100644 index 00000000000..d84e8d752dc --- /dev/null +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + class DiffStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_diff = nil + + @rpc_response.each do |diff_msg| + if current_diff.nil? + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params[:patch] = diff_msg.raw_patch_data + + current_diff = GitalyClient::Diff.new(diff_params) + else + current_diff.patch += diff_msg.raw_patch_data + end + + if diff_msg.end_of_patch + yield current_diff + current_diff = nil + end + end + end + end + end +end diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 890bd9a3554..b1dbf554e41 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -14,7 +14,7 @@ module Gitlab end def valid? - raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects") end def repos diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1b43440673c..ab38c0c3e34 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -95,7 +95,7 @@ module Gitlab labels = import_issue_labels(raw_issue) assignee_id = nil - if raw_issue.has_key?("owner") + if raw_issue.key?("owner") username = user_map[raw_issue["owner"]["name"]] if username.start_with?("@") @@ -144,7 +144,7 @@ module Gitlab def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift - next if raw_comment.has_key?("deletedBy") + next if raw_comment.key?("deletedBy") content = format_content(raw_comment["content"]) updates = format_updates(raw_comment["updates"]) @@ -235,15 +235,15 @@ module Gitlab def format_updates(raw_updates) updates = [] - if raw_updates.has_key?("status") + if raw_updates.key?("status") updates << "*Status: #{raw_updates["status"]}*" end - if raw_updates.has_key?("owner") + if raw_updates.key?("owner") updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end - if raw_updates.has_key?("cc") + if raw_updates.key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -255,7 +255,7 @@ module Gitlab updates << "*Cc: #{cc.join(", ")}*" end - if raw_updates.has_key?("labels") + if raw_updates.key?("labels") labels = raw_updates["labels"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -267,11 +267,11 @@ module Gitlab updates << "*Labels: #{labels.join(", ")}*" end - if raw_updates.has_key?("mergedInto") + if raw_updates.key?("mergedInto") updates << "*Merged into: ##{raw_updates["mergedInto"]}*" end - if raw_updates.has_key?("blockedOn") + if raw_updates.key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end @@ -279,7 +279,7 @@ module Gitlab updates << "*Blocked on: #{blocked_ons.join(", ")}*" end - if raw_updates.has_key?("blocking") + if raw_updates.key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb new file mode 100644 index 00000000000..b3c759b4730 --- /dev/null +++ b/lib/gitlab/health_checks/prometheus_text_format.rb @@ -0,0 +1,40 @@ +module Gitlab + module HealthChecks + class PrometheusTextFormat + def marshal(metrics) + "#{metrics_with_type_declarations(metrics).join("\n")}\n" + end + + private + + def metrics_with_type_declarations(metrics) + type_declaration_added = {} + + metrics.flat_map do |metric| + metric_lines = [] + + unless type_declaration_added.key?(metric.name) + type_declaration_added[metric.name] = true + metric_lines << metric_type_declaration(metric) + end + + metric_lines << metric_text(metric) + end + end + + def metric_type_declaration(metric) + "# TYPE #{metric.name} gauge" + end + + def metric_text(metric) + labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || '' + + if labels.empty? + "#{metric.name} #{metric.value}" + else + "#{metric.name}{#{labels}} #{metric.value}" + end + end + end + end +end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5ab3eeb3aff..f7ac48f7dbd 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -5,7 +5,10 @@ module Gitlab AVAILABLE_LANGUAGES = { 'en' => 'English', 'es' => 'Español', - 'de' => 'Deutsch' + 'de' => 'Deutsch', + 'zh_CN' => '简体中文', + 'zh_HK' => '繁體中文(香港)', + 'zh_TW' => '繁體中文(臺灣)' }.freeze def available_locales diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d0f3cf2b514..ff2b1d08c3c 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -38,6 +38,7 @@ project_tree: - notes: - :author - :events + - :stages - :statuses - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e23a4715f..695852526cb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,6 +3,7 @@ module Gitlab class RelationFactory OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 2d5e47a6f3b..5e299e26c54 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -41,11 +41,6 @@ module Gitlab def update_user_attributes if persisted? - if auth_hash.has_email? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) @@ -55,10 +50,6 @@ module Gitlab # For an existing identity with no change in DN, this line changes nothing. identity.extern_uid = auth_hash.uid end - - gl_user.ldap_email = auth_hash.has_email? - - gl_user end def changed? @@ -69,6 +60,10 @@ module Gitlab ldap_config.block_auto_created_users end + def sync_email_from_provider? + true + end + def allowed? Gitlab::LDAP::Access.allowed?(gl_user) end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cb8db2f1e9f..4779755bb22 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,161 +1,10 @@ module Gitlab module Metrics - extend Gitlab::CurrentSettings - - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def self.settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + extend Gitlab::Metrics::InfluxDb + extend Gitlab::Metrics::Prometheus def self.enabled? - settings[:enabled] || false - end - - def self.mri? - RUBY_ENGINE == 'ruby' - end - - def self.method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - @method_call_threshold ||= settings[:method_call_threshold] - end - - def self.pool - @pool - end - - def self.submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError - end - end - end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - - def self.prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def self.escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def self.measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) * 1000.0 - cpu_time = cpu_stop - cpu_start - - trans.increment("#{name}_real_time", real_time) - trans.increment("#{name}_cpu_time", cpu_time) - trans.increment("#{name}_call_count", 1) - - retval - end - - # Adds a tag to the current transaction (if any) - # - # name - The name of the tag to add. - # value - The value of the tag. - def self.tag_transaction(name, value) - trans = current_transaction - - trans&.add_tag(name, value) - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def self.action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def self.add_event(*args) - trans = current_transaction - - trans&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def self.series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - if enabled? - @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client. - new(udp: { host: host, port: port }) - end + influx_metrics_enabled? || prometheus_metrics_enabled? end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb new file mode 100644 index 00000000000..3a39791edbf --- /dev/null +++ b/lib/gitlab/metrics/influx_db.rb @@ -0,0 +1,170 @@ +module Gitlab + module Metrics + module InfluxDb + extend Gitlab::CurrentSettings + extend self + + MUTEX = Mutex.new + private_constant :MUTEX + + def influx_metrics_enabled? + settings[:enabled] || false + end + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 + } + end + + def mri? + RUBY_ENGINE == 'ruby' + end + + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= settings[:method_call_threshold] + end + + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end + end + end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) + end + + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def escape_value(value) + value.to_s.gsub('=', '\\=') + end + + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def tag_transaction(name, value) + trans = current_transaction + + trans&.add_tag(name, value) + end + + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction + + trans&.action = action + end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction + + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client. + new(udp: { host: host, port: port }) + end + end + end + @pool + end + end + end + end +end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb new file mode 100644 index 00000000000..3b5a2907195 --- /dev/null +++ b/lib/gitlab/metrics/null_metric.rb @@ -0,0 +1,10 @@ +module Gitlab + module Metrics + # Mocks ::Prometheus::Client::Metric and all derived metrics + class NullMetric + def method_missing(name, *args, &block) + nil + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb new file mode 100644 index 00000000000..60686509332 --- /dev/null +++ b/lib/gitlab/metrics/prometheus.rb @@ -0,0 +1,41 @@ +require 'prometheus/client' + +module Gitlab + module Metrics + module Prometheus + include Gitlab::CurrentSettings + + def prometheus_metrics_enabled? + @prometheus_metrics_enabled ||= current_application_settings[:prometheus_metrics_enabled] || false + end + + def registry + @registry ||= ::Prometheus::Client.registry + end + + def counter(name, docstring, base_labels = {}) + provide_metric(name) || registry.counter(name, docstring, base_labels) + end + + def summary(name, docstring, base_labels = {}) + provide_metric(name) || registry.summary(name, docstring, base_labels) + end + + def gauge(name, docstring, base_labels = {}) + provide_metric(name) || registry.gauge(name, docstring, base_labels) + end + + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets) + end + + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.new + end + end + end + end +end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 9ad7a38d505..ac9d66c836d 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -22,7 +22,11 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - Gitlab::LDAP::Config.new(name).options + if Gitlab::LDAP::Config.valid_provider?(name) + Gitlab::LDAP::Config.new(name).options + else + nil + end else Gitlab.config.omniauth.providers.find { |provider| provider.name == name } end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index afd24b4dcc5..7307f8c2c87 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,6 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash + update_email end def persisted? @@ -174,6 +175,22 @@ module Gitlab } end + def sync_email_from_provider? + auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + end + + def update_email + if auth_hash.has_email? && sync_email_from_provider? + if persisted? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end + + gl_user.external_email = true + gl_user.email_provider = auth_hash.provider + end + end + def log Gitlab::AppLogger end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb new file mode 100644 index 00000000000..0d541935bc6 --- /dev/null +++ b/lib/gitlab/otp_key_rotator.rb @@ -0,0 +1,87 @@ +module Gitlab + # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute. + # + # When +otp_key_base+ is changed, it invalidates the current encrypted values + # of User#otp_secret. This class can be used to decrypt all the values with + # the old key, encrypt them with the new key, and and update the database + # with the new values. + # + # For persistence between runs, a CSV file is used with the following columns: + # + # user_id, old_value, new_value + # + # Only the encrypted values are stored in this file. + # + # As users may have their 2FA settings changed at any time, this is only + # guaranteed to be safe if run offline. + class OtpKeyRotator + HEADERS = %w[user_id old_value new_value].freeze + + attr_reader :filename + + # Create a new rotator. +filename+ is used to store values by +calculate!+, + # and to update the database with new and old values in +apply!+ and + # +rollback!+, respectively. + def initialize(filename) + @filename = filename + end + + def rotate!(old_key:, new_key:) + old_key ||= Gitlab::Application.secrets.otp_key_base + + raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key + raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64 + + write_csv do |csv| + ActiveRecord::Base.transaction do + User.with_two_factor.in_batches do |relation| + rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) + rows.each do |row| + user = %i[id ciphertext iv salt].zip(row).to_h + new_value = reencrypt(user, old_key, new_key) + + User.where(id: user[:id]).update_all(encrypted_otp_secret: new_value) + csv << [user[:id], user[:ciphertext], new_value] + end + end + end + end + end + + def rollback! + ActiveRecord::Base.transaction do + CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| + User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value']) + end + end + end + + private + + attr_reader :old_key, :new_key + + def otp_secret_settings + @otp_secret_settings ||= User.encrypted_attributes[:otp_secret] + end + + def reencrypt(user, old_key, new_key) + original = user[:ciphertext].unpack("m").join + opts = { + iv: user[:iv].unpack("m").join, + salt: user[:salt].unpack("m").join, + algorithm: otp_secret_settings[:algorithm], + insecure_mode: otp_secret_settings[:insecure_mode] + } + + decrypted = Encryptor.decrypt(original, opts.merge(key: old_key)) + encrypted = Encryptor.encrypt(decrypted, opts.merge(key: new_key)) + [encrypted].pack("m") + end + + def write_csv(&blk) + File.open(filename, "w") do |file| + yield CSV.new(file, headers: HEADERS, write_headers: false) + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 1c0abc9f7cf..9ff6829cd49 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -80,6 +80,7 @@ module Gitlab # By rejecting `badges` the router can _count_ on the fact that `badges` will # be preceded by the `namespace/project`. PROJECT_WILDCARD_ROUTES = %w[ + - badges blame blob diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 36791fae60f..877aa6e6a28 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -25,8 +25,8 @@ module Gitlab def parse_entry(entry) raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) - raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') - raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public') source_pattern = entry['source'] public_path = entry['public'] diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb new file mode 100644 index 00000000000..36d1a8a6f64 --- /dev/null +++ b/lib/gitlab/routes/legacy_builds.rb @@ -0,0 +1,36 @@ +module Gitlab + module Routes + class LegacyBuilds + def initialize(map) + @map = map + end + + def draw + @map.instance_eval do + resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + resources :artifacts, only: [], controller: 'build_artifacts' do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end + end + end + + member do + get :raw + end + + resource :artifacts, only: [], controller: 'build_artifacts' do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + end + end + end + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9ce13feb79a..c81dc7e30d0 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -18,12 +18,6 @@ module Gitlab false end - def self.http_credentials_for_user(user) - return {} unless user.respond_to?(:username) - - { user: user.username } - end - def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 4c395b4266e..fa182c4deda 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -21,5 +21,13 @@ module Gitlab nil end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2e31f4462f9..85da4c8660b 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -83,7 +83,7 @@ module Gitlab end def valid_level?(level) - options.has_value?(level) + options.value?(level) end def level_name(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index fe37e4da94f..7f27317775c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -31,8 +31,7 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 - false + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) when 'info_refs' @@ -130,7 +129,7 @@ module Gitlab 'MaxSessionTime' => terminal[:max_session_time] } } - details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem) details end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 6e351365de0..c5f93336346 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index b341b5a0309..295c79fccfc 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -86,7 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/system_check.rb b/lib/system_check.rb new file mode 100644 index 00000000000..466c39904fa --- /dev/null +++ b/lib/system_check.rb @@ -0,0 +1,21 @@ +# Library to perform System Checks +# +# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck +# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor +# +# This structure decouples checks from Rake tasks and facilitates unit-testing +module SystemCheck + # Executes a bunch of checks for specified component + # + # @param [String] component name of the component relative to the checks being executed + # @param [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + def self.run(component, checks = []) + executor = SimpleExecutor.new(component) + + checks.each do |check| + executor << check + end + + executor.execute + end +end diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb new file mode 100644 index 00000000000..1d72c8d6903 --- /dev/null +++ b/lib/system_check/app/active_users_check.rb @@ -0,0 +1,17 @@ +module SystemCheck + module App + class ActiveUsersCheck < SystemCheck::BaseCheck + set_name 'Active users:' + + def multi_check + active_users = User.active.count + + if active_users > 0 + $stdout.puts active_users.to_s.color(:green) + else + $stdout.puts active_users.to_s.color(:red) + end + end + end + end +end diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb new file mode 100644 index 00000000000..d1fae192350 --- /dev/null +++ b/lib/system_check/app/database_config_exists_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class DatabaseConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Database config exists?' + + def check? + database_config_file = Rails.root.join('config', 'database.yml') + + File.exist?(database_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/database.yml.<your db> to config/database.yml', + 'Check that the information in config/database.yml is correct' + ) + for_more_information( + 'doc/install/databases.md', + 'http://guides.rubyonrails.org/getting_started.html#configuring-a-database' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb new file mode 100644 index 00000000000..198867f7ac6 --- /dev/null +++ b/lib/system_check/app/git_config_check.rb @@ -0,0 +1,42 @@ +module SystemCheck + module App + class GitConfigCheck < SystemCheck::BaseCheck + OPTIONS = { + 'core.autocrlf' => 'input' + }.freeze + + set_name 'Git configured correctly?' + + def check? + correct_options = OPTIONS.map do |name, value| + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + end + + correct_options.all? + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successful (according to their exit code) + # Returns false if any or all subcommands failed. + def repair! + return false unless is_gitlab_user? + + command_success = OPTIONS.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + end + + def show_error + try_fixing_it( + sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"") + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + end + end + end +end diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb new file mode 100644 index 00000000000..c388682dfb4 --- /dev/null +++ b/lib/system_check/app/git_version_check.rb @@ -0,0 +1,29 @@ +module SystemCheck + module App + class GitVersionCheck < SystemCheck::BaseCheck + set_name -> { "Git version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 7, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" + + try_fixing_it( + "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb new file mode 100644 index 00000000000..247aa0994e4 --- /dev/null +++ b/lib/system_check/app/gitlab_config_exists_check.rb @@ -0,0 +1,24 @@ +module SystemCheck + module App + class GitlabConfigExistsCheck < SystemCheck::BaseCheck + set_name 'GitLab config exists?' + + def check? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + + File.exist?(gitlab_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb new file mode 100644 index 00000000000..c609e48e133 --- /dev/null +++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb @@ -0,0 +1,30 @@ +module SystemCheck + module App + class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck + set_name 'GitLab config up to date?' + set_skip_reason "can't check because of previous errors" + + def skip? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + !File.exist?(gitlab_config_file) + end + + def check? + # omniauth or ldap could have been deleted from the file + !Gitlab.config['git_host'] + end + + def show_error + try_fixing_it( + 'Back-up your config/gitlab.yml', + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb new file mode 100644 index 00000000000..d246e058e86 --- /dev/null +++ b/lib/system_check/app/init_script_exists_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class InitScriptExistsCheck < SystemCheck::BaseCheck + set_name 'Init script exists?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def check? + script_path = '/etc/init.d/gitlab' + File.exist?(script_path) + end + + def show_error + try_fixing_it( + 'Install the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb new file mode 100644 index 00000000000..015c7ed1731 --- /dev/null +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module App + class InitScriptUpToDateCheck < SystemCheck::BaseCheck + SCRIPT_PATH = '/etc/init.d/gitlab'.freeze + + set_name 'Init script up-to-date?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def multi_check + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + + unless File.exist?(SCRIPT_PATH) + $stdout.puts "can't check because of previous errors".color(:magenta) + return + end + + recipe_content = File.read(recipe_path) + script_content = File.read(SCRIPT_PATH) + + if recipe_content == script_content + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + + def show_error + try_fixing_it( + 'Re-download the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb new file mode 100644 index 00000000000..3e0c436d6ee --- /dev/null +++ b/lib/system_check/app/log_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class LogWritableCheck < SystemCheck::BaseCheck + set_name 'Log directory writable?' + + def check? + File.writable?(log_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{log_path}", + "sudo chmod -R u+rwX #{log_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def log_path + Rails.root.join('log') + end + end + end +end diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb new file mode 100644 index 00000000000..5eedbacce77 --- /dev/null +++ b/lib/system_check/app/migrations_are_up_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class MigrationsAreUpCheck < SystemCheck::BaseCheck + set_name 'All migrations up?' + + def check? + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) + + migration_status !~ /down\s+\d{14}/ + end + + def show_error + try_fixing_it( + sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb new file mode 100644 index 00000000000..2b46d36fe51 --- /dev/null +++ b/lib/system_check/app/orphaned_group_members_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class OrphanedGroupMembersCheck < SystemCheck::BaseCheck + set_name 'Database contains orphaned GroupMembers?' + set_check_pass 'no' + set_check_fail 'yes' + + def check? + !GroupMember.where('user_id not in (select id from users)').exists? + end + + def show_error + try_fixing_it( + 'You can delete the orphaned records using something along the lines of:', + sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") + ) + end + end + end +end diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb new file mode 100644 index 00000000000..a6ec9f7665c --- /dev/null +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -0,0 +1,37 @@ +module SystemCheck + module App + class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck + set_name 'Projects have namespace:' + set_skip_reason "can't check, you have no projects" + + def skip? + !Project.exists? + end + + def multi_check + $stdout.puts '' + + Project.find_each(batch_size: 100) do |project| + $stdout.print sanitized_message(project) + + if project.namespace + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + end + + def show_error + try_fixing_it( + "Migrate global projects" + ) + for_more_information( + "doc/update/5.4-to-6.0.md in section \"#global-projects\"" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb new file mode 100644 index 00000000000..a0610e73576 --- /dev/null +++ b/lib/system_check/app/redis_version_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class RedisVersionCheck < SystemCheck::BaseCheck + MIN_REDIS_VERSION = '2.8.0'.freeze + set_name "Redis version >= #{MIN_REDIS_VERSION}?" + + def check? + redis_version = run_command(%w(redis-cli --version)) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) + + redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION)) + end + + def show_error + try_fixing_it( + "Update your redis server to a version >= #{MIN_REDIS_VERSION}" + ) + for_more_information( + 'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb new file mode 100644 index 00000000000..fd82f5f8a4a --- /dev/null +++ b/lib/system_check/app/ruby_version_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class RubyVersionCheck < SystemCheck::BaseCheck + set_name -> { "Ruby version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 3, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + try_fixing_it( + "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb new file mode 100644 index 00000000000..99a75e57abf --- /dev/null +++ b/lib/system_check/app/tmp_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class TmpWritableCheck < SystemCheck::BaseCheck + set_name 'Tmp directory writable?' + + def check? + File.writable?(tmp_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{tmp_path}", + "sudo chmod -R u+rwX #{tmp_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def tmp_path + Rails.root.join('tmp') + end + end + end +end diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb new file mode 100644 index 00000000000..7026d0ba075 --- /dev/null +++ b/lib/system_check/app/uploads_directory_exists_check.rb @@ -0,0 +1,21 @@ +module SystemCheck + module App + class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck + set_name 'Uploads directory exists?' + + def check? + File.directory?(Rails.root.join('public/uploads')) + end + + def show_error + try_fixing_it( + "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb new file mode 100644 index 00000000000..7df6c060254 --- /dev/null +++ b/lib/system_check/app/uploads_path_permission_check.rb @@ -0,0 +1,36 @@ +module SystemCheck + module App + class UploadsPathPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory has correct permissions?' + set_skip_reason 'skipped (no uploads folder found)' + + def skip? + !File.directory?(rails_uploads_path) + end + + def check? + File.stat(uploads_fullpath).mode == 040700 + end + + def show_error + try_fixing_it( + "sudo chmod 700 #{uploads_fullpath}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def rails_uploads_path + Rails.root.join('public/uploads') + end + + def uploads_fullpath + File.realpath(rails_uploads_path) + end + end + end +end diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb new file mode 100644 index 00000000000..b276a81eac1 --- /dev/null +++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb @@ -0,0 +1,40 @@ +module SystemCheck + module App + class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory tmp has correct permissions?' + set_skip_reason 'skipped (no tmp uploads folder yet)' + + def skip? + !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp) + end + + def check? + # If tmp upload dir has incorrect permissions, assume others do as well + # Verify drwx------ permissions + File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) + end + + def show_error + try_fixing_it( + "sudo chown -R #{gitlab_user} #{uploads_fullpath}", + "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;", + "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def upload_path_tmp + File.join(uploads_fullpath, 'tmp') + end + + def uploads_fullpath + File.realpath(Rails.root.join('public/uploads')) + end + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb new file mode 100644 index 00000000000..5dcb3f0886b --- /dev/null +++ b/lib/system_check/base_check.rb @@ -0,0 +1,129 @@ +module SystemCheck + # Base class for Checks. You must inherit from here + # and implement the methods below when necessary + class BaseCheck + include ::SystemCheck::Helpers + + # Define a custom term for when check passed + # + # @param [String] term used when check passed (default: 'yes') + def self.set_check_pass(term) + @check_pass = term + end + + # Define a custom term for when check failed + # + # @param [String] term used when check failed (default: 'no') + def self.set_check_fail(term) + @check_fail = term + end + + # Define the name of the SystemCheck that will be displayed during execution + # + # @param [String] name of the check + def self.set_name(name) + @name = name + end + + # Define the reason why we skipped the SystemCheck + # + # This is only used if subclass implements `#skip?` + # + # @param [String] reason to be displayed + def self.set_skip_reason(reason) + @skip_reason = reason + end + + # Term to be displayed when check passed + # + # @return [String] term when check passed ('yes' if not re-defined in a subclass) + def self.check_pass + call_or_return(@check_pass) || 'yes' + end + + ## Term to be displayed when check failed + # + # @return [String] term when check failed ('no' if not re-defined in a subclass) + def self.check_fail + call_or_return(@check_fail) || 'no' + end + + # Name of the SystemCheck defined by the subclass + # + # @return [String] the name + def self.display_name + call_or_return(@name) || self.name + end + + # Skip reason defined by the subclass + # + # @return [String] the reason + def self.skip_reason + call_or_return(@skip_reason) || 'skipped' + end + + # Does the check support automatically repair routine? + # + # @return [Boolean] whether check implemented `#repair!` method or not + def can_repair? + self.class.instance_methods(false).include?(:repair!) + end + + def can_skip? + self.class.instance_methods(false).include?(:skip?) + end + + def is_multi_check? + self.class.instance_methods(false).include?(:multi_check) + end + + # Execute the check routine + # + # This is where you should implement the main logic that will return + # a boolean at the end + # + # You should not print any output to STDOUT here, use the specific methods instead + # + # @return [Boolean] whether check passed or failed + def check? + raise NotImplementedError + end + + # Execute a custom check that cover multiple unities + # + # When using multi_check you have to provide the output yourself + def multi_check + raise NotImplementedError + end + + # Prints troubleshooting instructions + # + # This is where you should print detailed information for any error found during #check? + # + # You may use helper methods to help format the output: + # + # @see #try_fixing_it + # @see #fix_and_rerun + # @see #for_more_infromation + def show_error + raise NotImplementedError + end + + # When implemented by a subclass, will attempt to fix the issue automatically + def repair! + raise NotImplementedError + end + + # When implemented by a subclass, will evaluate whether check should be skipped or not + # + # @return [Boolean] whether or not this check should be skipped + def skip? + raise NotImplementedError + end + + def self.call_or_return(input) + input.respond_to?(:call) ? input.call : input + end + private_class_method :call_or_return + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb new file mode 100644 index 00000000000..c42ae4fe4c4 --- /dev/null +++ b/lib/system_check/helpers.rb @@ -0,0 +1,75 @@ +require 'tasks/gitlab/task_helpers' + +module SystemCheck + module Helpers + include ::Gitlab::TaskHelpers + + # Display a message telling to fix and rerun the checks + def fix_and_rerun + $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) + end + + # Display a formatted list of references (documentation or links) where to find more information + # + # @param [Array<String>] sources one or more references (documentation or links) + def for_more_information(*sources) + $stdout.puts ' For more information see:'.color(:blue) + sources.each do |source| + $stdout.puts " #{source}" + end + end + + def see_installation_guide_section(section) + "doc/install/installation.md in section \"#{section}\"" + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Display a formatted list of instructions on how to fix the issue identified by the #check? + # + # @param [Array<String>] steps one or short sentences with help how to fix the issue + def try_fixing_it(*steps) + steps = steps.shift if steps.first.is_a?(Array) + + $stdout.puts ' Try fixing it:'.color(:blue) + steps.each do |step| + $stdout.puts " #{step}" + end + end + + def sanitized_message(project) + if should_sanitize? + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " + else + "#{project.name_with_namespace.color(:yellow)} ... " + end + end + + def should_sanitize? + if ENV['SANITIZE'] == 'true' + true + else + false + end + end + + def omnibus_gitlab? + Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' + end + + def sudo_gitlab(command) + "sudo -u #{gitlab_user} -H #{command}" + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb new file mode 100644 index 00000000000..dc2d4643a01 --- /dev/null +++ b/lib/system_check/simple_executor.rb @@ -0,0 +1,99 @@ +module SystemCheck + # Simple Executor is current default executor for GitLab + # It is a simple port from display logic in the old check.rake + # + # There is no concurrency level and the output is progressively + # printed into the STDOUT + # + # @attr_reader [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + # @attr_reader [String] component name of the component relative to the checks being executed + class SimpleExecutor + attr_reader :checks + attr_reader :component + + # @param [String] component name of the component relative to the checks being executed + def initialize(component) + raise ArgumentError unless component.is_a? String + + @component = component + @checks = Set.new + end + + # Add a check to be executed + # + # @param [BaseCheck] check class + def <<(check) + raise ArgumentError unless check < BaseCheck + @checks << check + end + + # Executes defined checks in the specified order and outputs confirmation or error information + def execute + start_checking(component) + + @checks.each do |check| + run_check(check) + end + + finished_checking(component) + end + + # Executes a single check + # + # @param [SystemCheck::BaseCheck] check_klass + def run_check(check_klass) + $stdout.print "#{check_klass.display_name} ... " + + check = check_klass.new + + # When implements skip method, we run it first, and if true, skip the check + if check.can_skip? && check.skip? + $stdout.puts check_klass.skip_reason.color(:magenta) + return + end + + # When implements a multi check, we don't control the output + if check.is_multi_check? + check.multi_check + return + end + + if check.check? + $stdout.puts check_klass.check_pass.color(:green) + else + $stdout.puts check_klass.check_fail.color(:red) + + if check.can_repair? + $stdout.print 'Trying to fix error automatically. ...' + if check.repair! + $stdout.puts 'Success'.color(:green) + return + else + $stdout.puts 'Failed'.color(:red) + end + end + + check.show_error + end + end + + private + + # Prints header content for the series of checks to be executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Prints footer content for the series of checks executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + end +end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 0aa21a4bd13..b27f7475115 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -11,4 +11,12 @@ namespace :gettext do "{#{folders}}/**/*.{#{exts}}" ) end + + task :compile do + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 + FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) + + Rake::Task['gettext:pack'].invoke + Rake::Task['gettext:po_to_json'].invoke + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f41c73154f5..63c5e9b9c83 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,5 +1,9 @@ +# Temporary hack, until we migrate all checks to SystemCheck format +require 'system_check' +require 'system_check/helpers' + namespace :gitlab do - desc "GitLab | Check the configuration of GitLab and its environment" + desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check gitlab:incoming_email:check @@ -7,331 +11,38 @@ namespace :gitlab do gitlab:app:check} namespace :app do - desc "GitLab | Check the configuration of the GitLab Rails app" + desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :environment do warn_user_is_not_gitlab - start_checking "GitLab" - - check_git_config - check_database_config_exists - check_migrations_are_up - check_orphaned_group_members - check_gitlab_config_exists - check_gitlab_config_not_outdated - check_log_writable - check_tmp_writable - check_uploads - check_init_script_exists - check_init_script_up_to_date - check_projects_have_namespace - check_redis_version - check_ruby_version - check_git_version - check_active_users - - finished_checking "GitLab" - end - - # Checks - ######################## - - def check_git_config - print "Git configured with autocrlf=input? ... " - - options = { - "core.autocrlf" => "input" - } - - correct_options = options.map do |name, value| - run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value - end - - if correct_options.all? - puts "yes".color(:green) - else - print "Trying to fix Git error automatically. ..." - - if auto_fix_git_config(options) - puts "Success".color(:green) - else - puts "Failed".color(:red) - try_fixing_it( - sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - end - end - end - - def check_database_config_exists - print "Database config exists? ... " - - database_config_file = Rails.root.join("config", "database.yml") - - if File.exist?(database_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/database.yml.<your db> to config/database.yml", - "Check that the information in config/database.yml is correct" - ) - for_more_information( - see_database_guide, - "http://guides.rubyonrails.org/getting_started.html#configuring-a-database" - ) - fix_and_rerun - end - end - - def check_gitlab_config_exists - print "GitLab config exists? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - - if File.exist?(gitlab_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_gitlab_config_not_outdated - print "GitLab config outdated? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exist?(gitlab_config_file) - puts "can't check because of previous errors".color(:magenta) - end - - # omniauth or ldap could have been deleted from the file - unless Gitlab.config['git_host'] - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Backup your config/gitlab.yml", - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_init_script_exists - print "Init script exists? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - script_path = "/etc/init.d/gitlab" - - if File.exist?(script_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Install the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_init_script_up_to_date - print "Init script up-to-date? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") - script_path = "/etc/init.d/gitlab" - - unless File.exist?(script_path) - puts "can't check because of previous errors".color(:magenta) - return - end - - recipe_content = File.read(recipe_path) - script_content = File.read(script_path) - - if recipe_content == script_content - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Redownload the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_migrations_are_up - print "All migrations up? ... " - - migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) - - unless migration_status =~ /down\s+\d{14}/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") - ) - fix_and_rerun - end - end - - def check_orphaned_group_members - print "Database contains orphaned GroupMembers? ... " - if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".color(:red) - try_fixing_it( - "You can delete the orphaned records using something along the lines of:", - sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") - ) - else - puts "no".color(:green) - end - end - - def check_log_writable - print "Log directory writable? ... " - - log_path = Rails.root.join("log") - - if File.writable?(log_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{log_path}", - "sudo chmod -R u+rwX #{log_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - def check_tmp_writable - print "Tmp directory writable? ... " - - tmp_path = Rails.root.join("tmp") - - if File.writable?(tmp_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{tmp_path}", - "sudo chmod -R u+rwX #{tmp_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_uploads - print "Uploads directory setup correctly? ... " - - unless File.directory?(Rails.root.join('public/uploads')) - puts "no".color(:red) - try_fixing_it( - "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - return - end - - upload_path = File.realpath(Rails.root.join('public/uploads')) - upload_path_tmp = File.join(upload_path, 'tmp') - - if File.stat(upload_path).mode == 040700 - unless Dir.exist?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.color(:magenta) - return - end - - # If tmp upload dir has incorrect permissions, assume others do as well - # Verify drwx------ permissions - if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R #{gitlab_user} #{upload_path}", - "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod 700 #{upload_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_redis_version - min_redis_version = "2.8.0" - print "Redis version >= #{min_redis_version}? ... " - - redis_version = run_command(%w(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) - if redis_version && - (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your redis server to a version >= #{min_redis_version}" - ) - for_more_information( - "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq" - ) - fix_and_rerun - end + checks = [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck, + SystemCheck::App::ActiveUsersCheck + ] + + SystemCheck.run('GitLab', checks) end end namespace :gitlab_shell do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -513,33 +224,6 @@ namespace :gitlab do end end - def check_projects_have_namespace - print "projects have namespace: ... " - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - - if project.namespace - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Migrate global projects" - ) - for_more_information( - "doc/update/5.4-to-6.0.md in section \"#global-projects\"" - ) - fix_and_rerun - end - end - end - # Helper methods ######################## @@ -565,6 +249,8 @@ namespace :gitlab do end namespace :sidekiq do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -623,6 +309,8 @@ namespace :gitlab do end namespace :incoming_email do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab @@ -757,6 +445,8 @@ namespace :gitlab do end namespace :ldap do + include SystemCheck::Helpers + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -812,6 +502,8 @@ namespace :gitlab do end namespace :repo do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -826,6 +518,8 @@ namespace :gitlab do end namespace :user do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) @@ -848,55 +542,6 @@ namespace :gitlab do # Helper methods ########################## - def fix_and_rerun - puts " Please fix the error above and rerun the checks.".color(:red) - end - - def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - - puts " For more information see:".color(:blue) - sources.each do |source| - puts " #{source}" - end - end - - def finished_checking(component) - puts "" - puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" - puts "" - end - - def see_database_guide - "doc/install/databases.md" - end - - def see_installation_guide_section(section) - "doc/install/installation.md in section \"#{section}\"" - end - - def sudo_gitlab(command) - "sudo -u #{gitlab_user} -H #{command}" - end - - def gitlab_user - Gitlab.config.gitlab.user - end - - def start_checking(component) - puts "Checking #{component.color(:yellow)} ..." - puts "" - end - - def try_fixing_it(*steps) - steps = steps.shift if steps.first.is_a?(Array) - - puts " Try fixing it:".color(:blue) - steps.each do |step| - puts " #{step}" - end - end - def check_gitlab_shell required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) @@ -909,65 +554,6 @@ namespace :gitlab do end end - def check_ruby_version - required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) - - print "Ruby version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your ruby to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_git_version - required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) - - puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" - print "Git version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your git to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_active_users - puts "Active users: #{User.active.count}" - end - - def omnibus_gitlab? - Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' - end - - def sanitized_message(project) - if should_sanitize? - "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " - else - "#{project.name_with_namespace.color(:yellow)} ... " - end - end - - def should_sanitize? - if ENV['SANITIZE'] == "true" - true - else - false - end - end - def check_repo_integrity(repo_dir) puts "\nChecking repo at #{repo_dir.color(:yellow)}" diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e3c9d3b491c..964aa0fe1bc 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -98,34 +98,30 @@ module Gitlab end end + def gitlab_user + Gitlab.config.gitlab.user + end + + def is_gitlab_user? + return @is_gitlab_user unless @is_gitlab_user.nil? + + current_user = run_command(%w(whoami)).chomp + @is_gitlab_user = current_user == gitlab_user + end + def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user + return if @warned_user_not_gitlab + + unless is_gitlab_user? current_user = run_command(%w(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" - command_success.all? - else - false + @warned_user_not_gitlab = true end end diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index fc0ccc726ed..7728c485e8d 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -19,5 +19,21 @@ namespace :gitlab do puts "There are currently no users with 2FA enabled.".color(:yellow) end end + + namespace :rotate_key do + def rotator + @rotator ||= Gitlab::OtpKeyRotator.new(ENV['filename']) + end + + desc "Encrypt user OTP secrets with a new encryption key" + task apply: :environment do |t, args| + rotator.rotate!(old_key: ENV['old_key'], new_key: ENV['new_key']) + end + + desc "Rollback to secrets encrypted with the old encryption key" + task rollback: :environment do + rotator.rollback! + end + end end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index bc76d7edc55..50b8e331469 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -37,7 +37,7 @@ class GithubImport end def import! - @project.import_start + @project.force_import_start timings = Benchmark.measure do Github::Import.new(@project, @options).execute |