diff options
| author | Clement Ho <ClemMakesApps@gmail.com> | 2017-06-09 22:52:37 +0300 |
|---|---|---|
| committer | Clement Ho <ClemMakesApps@gmail.com> | 2017-06-09 22:52:37 +0300 |
| commit | f08a8ae18f74ad086695d62ff78ada2796e65829 (patch) | |
| tree | f65d9be5c66eb0d1ad8ceb1c13eb5c0da6bc2496 /lib | |
| parent | 66bbf30ed8bb006d9a968693fef266c86ec2325f (diff) | |
| parent | abc61f260074663e5711d3814d9b7d301d07a259 (diff) | |
Merge commit 'abc61f260074663e5711d3814d9b7d301d07a259' into 9-3-stable
Diffstat (limited to 'lib')
218 files changed, 4297 insertions, 1994 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 6f5f4283937..88f91c07194 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -45,10 +45,9 @@ module API end before { allow_access_with_scope :api } - before { header['X-Frame-Options'] = 'SAMEORIGIN' } - before { Gitlab::I18n.set_locale(current_user) } + before { Gitlab::I18n.locale = current_user&.preferred_language } - after { Gitlab::I18n.reset_locale } + after { Gitlab::I18n.use_default_locale } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -95,6 +94,8 @@ module API mount ::API::DeployKeys mount ::API::Deployments mount ::API::Environments + mount ::API::Events + mount ::API::Features mount ::API::Files mount ::API::Groups mount ::API::Internal @@ -111,6 +112,7 @@ module API mount ::API::Notes mount ::API::NotificationSettings mount ::API::Pipelines + mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects mount ::API::ProjectSnippets diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 827a38d33da..10f2d5ef6a3 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -68,7 +68,14 @@ module API name = params[:name] || params[:context] || 'default' - pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) + pipeline = @project.pipeline_for(ref, commit.sha) + unless pipeline + pipeline = @project.pipelines.create!( + source: :external, + sha: commit.sha, + ref: ref, + user: current_user) + end status = GenericCommitStatus.running_or_pending.find_or_initialize_by( project: @project, diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 621b9dcecd9..c6fc17cc391 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -176,7 +176,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 8a54f7f3f05..7cdee8aced7 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -76,6 +76,27 @@ module API end end + desc 'Update an existing deploy key for a project' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + optional :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" + at_least_one_of :title, :can_push + end + put ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys.find(params.delete(:key_id)) + + authorize!(:update_deploy_key, key) + + if key.update_attributes(declared_params(include_missing: false)) + present key, with: Entities::SSHKey + else + render_validation_error!(key) + end + end + desc 'Enable a deploy key for a project' do detail 'This feature was added in GitLab 8.11' success Entities::SSHKey diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9f8304f7690..a836df3dc81 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -5,7 +5,10 @@ module API end class UserBasic < UserSafe - expose :id, :state, :avatar_url + expose :id, :state + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url do |user, options| Gitlab::Routing.url_helpers.user_url(user) @@ -50,14 +53,14 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at, :push_events, :tag_push_events + expose :id, :url, :created_at, :push_events, :tag_push_events, :repository_update_events expose :enable_ssl_verification end class ProjectHook < Hook expose :project_id, :issues_events, :merge_requests_events expose :note_events, :pipeline_events, :wiki_page_events - expose :build_events, as: :job_events + expose :job_events end class BasicProjectDetails < Grape::Entity @@ -97,7 +100,11 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :import_status + expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -141,7 +148,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -217,7 +226,7 @@ module API end class ProjectSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at @@ -227,7 +236,7 @@ module API end class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at @@ -248,7 +257,9 @@ module API class RepoDiff < Grape::Entity expose :old_path, :new_path, :a_mode, :b_mode, :diff - expose :new_file, :renamed_file, :deleted_file + expose :new_file?, as: :new_file + expose :renamed_file?, as: :renamed_file + expose :deleted_file?, as: :deleted_file end class Milestone < ProjectEntity @@ -322,7 +333,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -335,7 +346,7 @@ module API expose :commits, using: Entities::RepoCommit expose :diffs, using: Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -466,7 +477,7 @@ module API expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events expose :tag_push_events, :note_events, :pipeline_events - expose :build_events, as: :job_events + expose :job_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -539,7 +550,7 @@ module API end expose :diffs, using: Entities::RepoDiff do |compare, options| - compare.diffs(all_diffs: true).to_a + compare.diffs(limits: false).to_a end expose :compare_timeout do |compare, options| @@ -666,6 +677,7 @@ module API class Variable < Grape::Entity expose :key, :value + expose :protected?, as: :protected end class Pipeline < PipelineBasic @@ -677,6 +689,17 @@ module API expose :coverage end + class PipelineSchedule < Grape::Entity + expose :id + expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active + expose :created_at, :updated_at + expose :owner, using: Entities::UserBasic + end + + class PipelineScheduleDetails < PipelineSchedule + expose :last_pipeline, using: Entities::PipelineBasic + end + class EnvironmentBasic < Grape::Entity expose :id, :name, :slug, :external_url end @@ -733,6 +756,28 @@ module API expose :impersonation end + class FeatureGate < Grape::Entity + expose :key + expose :value + end + + class Feature < Grape::Entity + expose :name + expose :state + expose :gates, using: FeatureGate do |model| + model.gates.map do |gate| + value = model.gate_values[gate.key] + + # By default all gate values are populated. Only show relevant ones. + if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?) + next + end + + { key: gate.key, value: value } + end.compact + end + end + module JobRequest class JobInfo < Grape::Entity expose :name, :stage diff --git a/lib/api/events.rb b/lib/api/events.rb new file mode 100644 index 00000000000..dabdf579119 --- /dev/null +++ b/lib/api/events.rb @@ -0,0 +1,86 @@ +module API + class Events < Grape::API + include PaginationParams + + helpers do + params :event_filter_params do + optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' + optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' + optional :before, type: Date, desc: 'Include only events created before this date' + optional :after, type: Date, desc: 'Include only events created after this date' + end + + params :sort_params do + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return events sorted in ascending and descending order' + end + + def present_events(events) + events = events.reorder(created_at: params[:sort]) + + present paginate(events), with: Entities::Event + end + end + + resource :events do + desc "List currently authenticated user's events" do + detail 'This feature was introduced in GitLab 9.3.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get do + authenticate! + + events = EventsFinder.new(params.merge(source: current_user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: String, desc: 'The ID or Username of the user' + end + resource :users do + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ':id/events' do + user = find_user(params[:id]) + not_found!('User') unless user + + events = EventsFinder.new(params.merge(source: user, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc "List a Project's visible events" do + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + get ":id/events" do + events = EventsFinder.new(params.merge(source: user_project, current_user: current_user)).execute.preload(:author, :target) + + present_events(events) + end + end + end +end diff --git a/lib/api/features.rb b/lib/api/features.rb new file mode 100644 index 00000000000..cff0ba2ddff --- /dev/null +++ b/lib/api/features.rb @@ -0,0 +1,36 @@ +module API + class Features < Grape::API + before { authenticated_as_admin! } + + resource :features do + desc 'Get a list of all features' do + success Entities::Feature + end + get do + features = Feature.all + + present features, with: Entities::Feature, current_user: current_user + end + + desc 'Set the gate value for the given feature' do + success Entities::Feature + end + params do + requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time' + end + post ':name' do + feature = Feature.get(params[:name]) + + if %w(0 false).include?(params[:value]) + feature.disable + elsif params[:value] == 'true' + feature.enable + else + feature.enable_percentage_of_time(params[:value].to_i) + end + + present feature, with: Entities::Feature, current_user: current_user + end + end + end +end diff --git a/lib/api/files.rb b/lib/api/files.rb index e6ea12c5ab7..25b0968a271 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -10,7 +10,8 @@ module API file_content: attrs[:content], file_content_encoding: attrs[:encoding], author_email: attrs[:author_email], - author_name: attrs[:author_name] + author_name: attrs[:author_name], + last_commit_sha: attrs[:last_commit_id] } end @@ -46,6 +47,7 @@ module API use :simple_file_params requires :content, type: String, desc: 'File content' optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + optional :last_commit_id, type: String, desc: 'Last known commit id for this file' end end @@ -111,7 +113,12 @@ module API authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + + begin + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + rescue ::Files::UpdateService::FileChangedError => e + render_api_error!(e.message, 400) + end if result[:status] == :success status(200) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 2c09725601e..ebbaed0cbb7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -24,7 +24,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] @@ -83,7 +83,7 @@ module API group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute if group.persisted? - present group, with: Entities::Group, current_user: current_user + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to save group #{group.errors.messages}", 400) end @@ -101,8 +101,6 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - at_least_one_of :name, :path, :description, :visibility, - :lfs_enabled, :request_access_enabled end put ':id' do group = find_group!(params[:id]) @@ -151,8 +149,8 @@ module API end get ":id/projects" do group = find_group!(params[:id]) - projects = GroupProjectsFinder.new(group: group, current_user: current_user).execute - projects = filter_projects(projects) + projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project present paginate(projects), with: entity, current_user: current_user end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index c643ea8e5a7..2c73a6fdc4e 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -158,7 +158,7 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) + if params_hash[key].present? || (params_hash.key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end @@ -256,31 +256,21 @@ module API # project helpers - def filter_projects(projects) - if params[:membership] - projects = projects.merge(current_user.authorized_projects) - end - - if params[:owned] - projects = projects.merge(current_user.owned_projects) - end - - if params[:starred] - projects = projects.merge(current_user.starred_projects) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:visibility].present? - projects = projects.search_by_visibility(params[:visibility]) - end - - projects = projects.where(archived: params[:archived]) + def reorder_projects(projects) projects.reorder(params[:order_by] => params[:sort]) end + def project_finder_params + finder_params = {} + finder_params[:owned] = true if params[:owned].present? + finder_params[:non_public] = true if params[:membership].present? + finder_params[:starred] = true if params[:starred].present? + finder_params[:visibility_level] = Gitlab::VisibilityLevel.level_value(params[:visibility]) if params[:visibility] + finder_params[:archived] = params[:archived] + finder_params[:search] = params[:search] if params[:search] + finder_params + end + # file helpers def uploaded_file(field, uploads_path) @@ -301,7 +291,7 @@ module API UploadedFile.new( file_path, params["#{field}.name"], - params["#{field}.type"] || 'application/octet-stream', + params["#{field}.type"] || 'application/octet-stream' ) end @@ -321,6 +311,16 @@ module API end end + def present_artifacts!(artifacts_file) + return not_found! unless artifacts_file.exists? + + if artifacts_file.file_storage? + present_file!(artifacts_file.path, artifacts_file.filename) + else + redirect_to(artifacts_file.url) + end + end + private def private_token diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 264df7271a3..d3732d67622 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -42,6 +42,22 @@ module API @project, @wiki = Gitlab::RepoPath.parse(params[:project]) end end + + # Project id to pass between components that don't share/don't have + # access to the same filesystem mounts + def gl_repository + Gitlab::GlRepository.gl_repository(project, wiki?) + end + + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + def repository_path + if wiki? + project.wiki.repository.path_to_repo + else + project.repository.path_to_repo + end + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 2971887770b..38631953014 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -32,31 +32,23 @@ module API actor.update_last_used_at if actor.is_a?(Key) - access_checker = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_status = access_checker + access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess + access_checker = access_checker_klass .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) - .check(params[:action], params[:changes]) - response = { status: access_status.status, message: access_status.message } - - if access_status.status - log_user_activity(actor) - - # Project id to pass between components that don't share/don't have - # access to the same filesystem mounts - response[:gl_repository] = Gitlab::GlRepository.gl_repository(project, wiki?) - - # Return the repository full path so that gitlab-shell has it when - # handling ssh commands - response[:repository_path] = - if wiki? - project.wiki.repository.path_to_repo - else - project.repository.path_to_repo - end + begin + access_checker.check(params[:action], params[:changes]) + rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e + return { status: false, message: e.message } end - response + log_user_activity(actor) + + { + status: true, + gl_repository: gl_repository, + repository_path: repository_path + } end post "/lfs_authenticate" do @@ -90,7 +82,7 @@ module API { api_version: API.version, gitlab_version: Gitlab::VERSION, - gitlab_rev: Gitlab::REVISION, + gitlab_rev: Gitlab::REVISION } end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 0223957fde1..8a67de10bca 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -224,16 +224,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb new file mode 100644 index 00000000000..93d89209934 --- /dev/null +++ b/lib/api/pipeline_schedules.rb @@ -0,0 +1,131 @@ +module API + class PipelineSchedules < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all pipeline schedules' do + success Entities::PipelineSchedule + end + params do + use :pagination + optional :scope, type: String, values: %w[active inactive], + desc: 'The scope of pipeline schedules' + end + get ':id/pipeline_schedules' do + authorize! :read_pipeline_schedule, user_project + + schedules = PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope]) + .preload([:owner, :last_pipeline]) + present paginate(schedules), with: Entities::PipelineSchedule + end + + desc 'Get a single pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + get ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :read_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + present pipeline_schedule, with: Entities::PipelineScheduleDetails + end + + desc 'Create a new pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :description, type: String, desc: 'The description of pipeline schedule' + requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' + optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' + end + post ':id/pipeline_schedules' do + authorize! :create_pipeline_schedule, user_project + + pipeline_schedule = Ci::CreatePipelineScheduleService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + if pipeline_schedule.persisted? + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Edit a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + optional :description, type: String, desc: 'The description of pipeline schedule' + optional :ref, type: String, desc: 'The branch/tag name will be triggered' + optional :cron, type: String, desc: 'The cron' + optional :cron_timezone, type: String, desc: 'The timezone' + optional :active, type: Boolean, desc: 'The activation of pipeline schedule' + end + put ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.update(declared_params(include_missing: false)) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Take ownership of a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + post ':id/pipeline_schedules/:pipeline_schedule_id/take_ownership' do + authorize! :update_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + if pipeline_schedule.own!(current_user) + present pipeline_schedule, with: Entities::PipelineScheduleDetails + else + render_validation_error!(pipeline_schedule) + end + end + + desc 'Delete a pipeline schedule' do + success Entities::PipelineScheduleDetails + end + params do + requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' + end + delete ':id/pipeline_schedules/:pipeline_schedule_id' do + authorize! :admin_pipeline_schedule, user_project + + not_found!('PipelineSchedule') unless pipeline_schedule + + status :accepted + present pipeline_schedule.destroy, with: Entities::PipelineScheduleDetails + end + end + + helpers do + def pipeline_schedule + @pipeline_schedule ||= + user_project.pipeline_schedules + .preload(:owner, :last_pipeline) + .find_by(id: params.delete(:pipeline_schedule_id)) + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 9117704aa46..e505cae3992 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -47,7 +47,7 @@ module API new_pipeline = Ci::CreatePipelineService.new(user_project, current_user, declared_params(include_missing: false)) - .execute(ignore_skip_ci: true, save_on_errors: false) + .execute(:api, ignore_skip_ci: true, save_on_errors: false) if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 87dfd1573a4..7a345289617 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -54,7 +54,6 @@ module API end post ":id/hooks" do hook_params = declared_params(include_missing: false) - hook_params[:build_events] = hook_params.delete(:job_events) { false } hook = user_project.hooks.new(hook_params) @@ -78,7 +77,6 @@ module API hook = user_project.hooks.find(params.delete(:hook_id)) update_params = declared_params(include_missing: false) - update_params[:build_events] = update_params.delete(:job_events) if update_params[:job_events] if hook.update_attributes(update_params) present hook, with: Entities::ProjectHook diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 98bc9c28527..64efe82a937 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -49,6 +49,7 @@ module API requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' requires :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' requires :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' @@ -77,6 +78,7 @@ module API optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' optional :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 9a6cb43abf7..56046742e08 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -21,6 +21,7 @@ module API optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :tag_list, type: Array[String], desc: 'The list of tags for a project' end params :optional_params do @@ -58,6 +59,8 @@ module API optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' end params :create_params do @@ -65,16 +68,19 @@ module API optional :import_url, type: String, desc: 'URL from which the project is imported' end - def present_projects(projects, options = {}) + def present_projects(options = {}) + projects = ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute + projects = reorder_projects(projects) + projects = projects.with_statistics if params[:statistics] + projects = projects.with_issues_enabled if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] + options = options.reverse_merge( - with: Entities::Project, - current_user: current_user, - simple: params[:simple], + with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, + statistics: params[:statistics], + current_user: current_user ) - - projects = filter_projects(projects) - projects = projects.with_statistics if options[:statistics] - options[:with] = Entities::BasicProjectDetails if options[:simple] + options[:with] = Entities::BasicProjectDetails if params[:simple] present paginate(projects), options end @@ -88,8 +94,7 @@ module API use :statistics_params end get do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new(current_user: current_user).execute, with: entity, statistics: params[:statistics] + present_projects end desc 'Create new project' do @@ -104,7 +109,7 @@ module API end post do attrs = declared_params(include_missing: false) - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -124,6 +129,7 @@ module API params do requires :name, type: String, desc: 'The name of the project' requires :user_id, type: Integer, desc: 'The ID of a user' + optional :path, type: String, desc: 'The path of the repository' optional :default_branch, type: String, desc: 'The default branch of the project' use :optional_params use :create_params @@ -161,16 +167,6 @@ module API user_can_admin_project: can?(current_user, :admin_project, user_project), statistics: params[:statistics] end - desc 'Get events for a single project' do - success Entities::Event - end - params do - use :pagination - end - get ":id/events" do - present paginate(user_project.events.recent), with: Entities::Event - end - desc 'Fork new project for the current user or provided namespace.' do success Entities::Project end @@ -225,8 +221,9 @@ module API :request_access_enabled, :shared_runners_enabled, :snippets_enabled, + :tag_list, :visibility, - :wiki_enabled, + :wiki_enabled ] optional :name, type: String, desc: 'The name of the project' optional :default_branch, type: String, desc: 'The default branch of the project' @@ -241,7 +238,7 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? - attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.key?(:jobs_enabled) result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 8f16e532ecb..14d2bff9cb5 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -85,7 +85,7 @@ module API optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' optional :format, type: String, desc: 'The archive format' end - get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 6fbb02cb3aa..4552115b3e2 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -141,7 +141,7 @@ module API patch '/:id/trace' do job = authenticate_job! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -241,16 +241,7 @@ module API get '/:id/artifacts' do job = authenticate_job! - artifacts_file = job.artifacts_file - unless artifacts_file.file_storage? - return redirect_to job.artifacts_file.url - end - - unless artifacts_file.exists? - not_found! - end - - present_file!(artifacts_file.path, artifacts_file.filename) + present_artifacts!(job.artifacts_file) end end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 23ef62c2258..47bd9940f77 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -304,7 +304,13 @@ module API required: true, name: :url, type: String, - desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + desc: 'The base URL to the JIRA instance web interface which is being linked to this GitLab project. E.g., https://jira.example.com' + }, + { + required: false, + name: :api_url, + type: String, + desc: 'The base URL to the JIRA instance API. Web URL value will be used if not set. E.g., https://jira-api.example.com' }, { required: true, @@ -356,7 +362,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { @@ -559,7 +565,7 @@ module API SlackService, MattermostService, MicrosoftTeamsService, - TeamcityService, + TeamcityService ] if Rails.env.development? @@ -577,7 +583,7 @@ module API service_classes += [ MockCiService, MockDeploymentService, - MockMonitoringService, + MockMonitoringService ] end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 82f513c984e..25027c3b114 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -110,6 +110,7 @@ module API optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 53f5953a8fb..c630c24c339 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -58,6 +58,7 @@ module API requires :title, type: String, desc: 'The title of a snippet' requires :file_name, type: String, desc: 'The name of a snippet file' requires :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, default: 'internal', @@ -85,6 +86,7 @@ module API optional :title, type: String, desc: 'The title of a snippet' optional :file_name, type: String, desc: 'The name of a snippet file' optional :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index dbe54d3cd31..91567909998 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -5,7 +5,7 @@ module API subscribable_types = { 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 05b4b490e27..df4632346dd 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,7 +5,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' + declared_params.key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/users.rb b/lib/api/users.rb index 40acaebf670..dda64715ee1 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -56,16 +56,7 @@ module API authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) - users = User.all - users = User.where(username: params[:username]) if params[:username] - users = users.active if params[:active] - users = users.search(params[:search]) if params[:search].present? - users = users.blocked if params[:blocked] - - if current_user.admin? - users = users.joins(:identities).merge(Identity.with_extern_uid(params[:provider], params[:extern_uid])) if params[:extern_uid] && params[:provider] - users = users.external if params[:external] - end + users = UsersFinder.new(current_user, params).execute entity = current_user.admin? ? Entities::UserPublic : Entities::UserBasic present paginate(users), with: entity @@ -133,10 +124,6 @@ module API optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes - at_least_one_of :email, :password, :name, :username, :skype, :linkedin, - :twitter, :website_url, :organization, :projects_limit, - :extern_uid, :provider, :bio, :location, :admin, - :can_create_group, :confirm, :external end put ":id" do authenticated_as_admin! @@ -295,13 +282,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserWorker.perform_async(current_user.id, user.id) + user.delete_async(deleted_by: current_user, params: params) end desc 'Block a user. Available only for admins.' @@ -336,27 +324,6 @@ module API end end - desc 'Get the contribution events of a specified user' do - detail 'This feature was introduced in GitLab 8.13.' - success Entities::Event - end - params do - requires :id, type: Integer, desc: 'The ID of the user' - use :pagination - end - get ':id/events' do - user = User.find_by(id: params[:id]) - not_found!('User') unless user - - events = user.events. - merge(ProjectsFinder.new(current_user: current_user).execute). - references(:project). - with_associations. - recent - - present paginate(events), with: Entities::Event - end - params do requires :user_id, type: Integer, desc: 'The ID of the user' end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 21935922414..93ad9eb26b8 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -225,16 +225,6 @@ module API find_build(id) || not_found! end - def present_artifacts!(artifacts_file) - if !artifacts_file.file_storage? - redirect_to(build.artifacts_file.url) - elsif artifacts_file.exists? - present_file!(artifacts_file.path, artifacts_file.filename) - else - not_found! - end - end - def filter_builds(builds, scope) return builds if scope.nil? || scope.empty? diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 674de592f0a..5936f4700aa 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -167,7 +167,7 @@ module API } if params[:path] - commit.raw_diffs(all_diffs: true).each do |diff| + commit.raw_diffs(limits: false).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb index bbb174b6003..b90e2061da3 100644 --- a/lib/api/v3/deploy_keys.rb +++ b/lib/api/v3/deploy_keys.rb @@ -41,6 +41,7 @@ module API params do requires :key, type: String, desc: 'The new deploy key' requires :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" end post ":id/#{path}" do params[:key].strip! diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 1c08e25c00c..7c5065dee90 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -69,7 +69,9 @@ module API expose :creator_id expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } @@ -129,7 +131,9 @@ module API class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level expose :lfs_enabled?, as: :lfs_enabled - expose :avatar_url + expose :avatar_url do |user, options| + user.avatar_url(only_path: false) + end expose :web_url expose :request_access_enabled expose :full_name, :full_path @@ -222,7 +226,7 @@ module API class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| - compare.raw_diffs(all_diffs: true).to_a + compare.raw_diffs(limits: false).to_a end end @@ -237,7 +241,8 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :build_events, :pipeline_events + expose :tag_push_events, :note_events, :pipeline_events + expose :job_events, as: :build_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -249,7 +254,8 @@ module API class ProjectHook < ::API::Entities::Hook expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :build_events, :pipeline_events, :wiki_page_events + expose :note_events, :pipeline_events, :wiki_page_events + expose :job_events, as: :build_events end class Issue < ::API::Entities::Issue diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb index 42922df6e29..2c52d21fa1c 100644 --- a/lib/api/v3/groups.rb +++ b/lib/api/v3/groups.rb @@ -20,7 +20,7 @@ module API def present_groups(groups, options = {}) options = options.reverse_merge( with: Entities::Group, - current_user: current_user, + current_user: current_user ) groups = groups.with_statistics if options[:statistics] diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb index 0f234d4cdad..d9e76560d03 100644 --- a/lib/api/v3/helpers.rb +++ b/lib/api/v3/helpers.rb @@ -14,6 +14,33 @@ module API authorize! access_level, merge_request merge_request end + + # project helpers + + def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:visibility].present? + projects = projects.where(visibility_level: Gitlab::VisibilityLevel.level_value(params[:visibility])) + end + + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) + end end end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 06cc704afc6..20976b9dd08 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -44,7 +44,7 @@ module API end def set_only_allow_merge_if_pipeline_succeeds! - if params.has_key?(:only_allow_merge_if_build_succeeds) + if params.key?(:only_allow_merge_if_build_succeeds) params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) end end @@ -88,7 +88,7 @@ module API options = options.reverse_merge( with: ::API::V3::Entities::Project, current_user: current_user, - simple: params[:simple], + simple: params[:simple] ) projects = filter_projects(projects) @@ -147,7 +147,7 @@ module API get '/starred' do authenticate! - present_projects current_user.viewable_starred_projects + present_projects ProjectsFinder.new(current_user: current_user, params: { starred: true }).execute end desc 'Get all projects for admin user' do diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index e4d14bc8168..0eaa0de2eef 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -72,7 +72,7 @@ module API optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' optional :format, type: String, desc: 'The archive format' end - get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 61629a04174..118c6df6549 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -377,7 +377,7 @@ module API name: :ca_pem, type: String, desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' - }, + } ], 'mattermost-slash-commands' => [ { diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb index 068750ec077..690768db82f 100644 --- a/lib/api/v3/subscriptions.rb +++ b/lib/api/v3/subscriptions.rb @@ -7,7 +7,7 @@ module API 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, - 'labels' => proc { |id| find_project_label(id) }, + 'labels' => proc { |id| find_project_label(id) } } params do diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb index 81ae4e8137d..d5b90e435ba 100644 --- a/lib/api/v3/time_tracking_endpoints.rb +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -6,7 +6,7 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.key?(:issue_id) ? 'issue' : 'merge_request' end def issuable_key diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 5acde41551b..381c4ef50b0 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -42,6 +42,7 @@ module API params do requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end post ':id/variables' do variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) @@ -59,13 +60,14 @@ module API params do optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: String, desc: 'Whether the variable is protected' end put ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - if variable.update(value: params[:value]) + if variable.update(declared_params(include_missing: false).except(:key)) present variable, with: Entities::Variable else render_validation_error!(variable) diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 51fa3867e67..1f4bda6f588 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', ArtifactUploader.artifacts_path) + super('artifacts', ArtifactUploader.local_artifacts_store) end def create_files_dir diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 6b29600a751..a1685c77916 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -7,15 +7,15 @@ module Backup prepare Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - if project.empty_repo? - $progress.puts "[SKIPPED]".color(:cyan) + if empty_repo?(project) + progress.puts "[SKIPPED]".color(:cyan) else in_path(path_to_project_repo) do |dir| FileUtils.mkdir_p(path_to_tars(project)) @@ -23,10 +23,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -34,12 +31,9 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(project, cmd.join(' '), output) end end @@ -48,19 +42,16 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_repo) - $progress.print " * #{wiki.path_with_namespace} ... " - if wiki.repository.empty? - $progress.puts " [SKIPPED]".color(:cyan) + progress.print " * #{wiki.path_with_namespace} ... " + if empty_repo?(wiki) + progress.puts " [SKIPPED]".color(:cyan) else cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Backup failed' + progress_warn(wiki, cmd.join(' '), output) end end end @@ -80,7 +71,7 @@ module Backup end Project.find_each(batch_size: 1000) do |project| - $progress.print " * #{project.path_with_namespace} ... " + progress.print " * #{project.path_with_namespace} ... " path_to_project_repo = path_to_repo(project) path_to_project_bundle = path_to_bundle(project) @@ -94,12 +85,9 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts "[DONE]".color(:green) + progress.puts "[DONE]".color(:green) else - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end in_path(path_to_tars(project)) do |dir| @@ -107,10 +95,7 @@ module Backup output, status = Gitlab::Popen.popen(cmd) unless status.zero? - puts "[FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end @@ -119,7 +104,7 @@ module Backup path_to_wiki_bundle = path_to_bundle(wiki) if File.exist?(path_to_wiki_bundle) - $progress.print " * #{wiki.path_with_namespace} ... " + progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then @@ -129,22 +114,19 @@ module Backup output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else - puts " [FAILED]".color(:red) - puts "failed: #{cmd.join(' ')}" - puts output - abort 'Restore failed' + progress_warn(project, cmd.join(' '), output) end end end - $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) + progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args output, status = Gitlab::Popen.popen(cmd) if status.zero? - $progress.puts " [DONE]".color(:green) + progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" @@ -201,8 +183,25 @@ module Backup private + def progress_warn(project, cmd, output) + progress.puts "[WARNING] Executing #{cmd}".color(:orange) + progress.puts "Ignoring error on #{project.path_with_namespace} - #{output}".color(:orange) + end + + def empty_repo?(project_or_wiki) + project_or_wiki.repository.empty_repo? + rescue => e + progress.puts "Ignoring repository error and continuing backing up project: #{project_or_wiki.path_with_namespace} - #{e.message}".color(:orange) + + false + end + def repository_storage_paths_args Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end + + def progress + $progress + end end end diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb new file mode 100644 index 00000000000..c9fcf057c5f --- /dev/null +++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb @@ -0,0 +1,13 @@ +module Banzai + module Filter + class AsciiDocPostProcessingFilter < HTML::Pipeline::Filter + def call + doc.search('[data-math-style]').each do |node| + node.set_attribute('class', 'code math js-render-math') + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 522217deae4..2d6e8ffc90f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -31,6 +31,10 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow data-math-style attribute in order to support LaTeX formatting + whitelist[:attributes]['code'] = %w(data-math-style) + whitelist[:attributes]['pre'] = %w(data-math-style) + # Allow html5 details/summary elements whitelist[:elements].push('details') whitelist[:elements].push('summary') diff --git a/lib/banzai/pipeline/ascii_doc_pipeline.rb b/lib/banzai/pipeline/ascii_doc_pipeline.rb new file mode 100644 index 00000000000..1048b927cd3 --- /dev/null +++ b/lib/banzai/pipeline/ascii_doc_pipeline.rb @@ -0,0 +1,14 @@ +module Banzai + module Pipeline + class AsciiDocPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::PlantumlFilter, + Filter::AsciiDocPostProcessingFilter + ] + end + end + end +end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index a20200c5879..1e2536231d8 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -163,14 +163,15 @@ module Banzai # been queried the object is returned from the cache. def collection_objects_for_ids(collection, ids) if RequestStore.active? + ids = ids.map(&:to_i) cache = collection_cache[collection_cache_key(collection)] - to_query = ids.map(&:to_i) - cache.keys + to_query = ids - cache.keys unless to_query.empty? collection.where(id: to_query).each { |row| cache[row.id] = row } end - cache.values + cache.values_at(*ids) else collection.where(id: ids) end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb index 4f8efe03bae..c52acbc3ddc 100644 --- a/lib/bitbucket/representation/pull_request_comment.rb +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -22,11 +22,11 @@ module Bitbucket end def inline? - raw.has_key?('inline') + raw.key?('inline') end def has_parent? - raw.has_key?('parent') + raw.key?('parent') end private diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b439b0ee29b..55402101e43 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -20,7 +20,7 @@ module Ci italic: 0x02, underline: 0x04, conceal: 0x08, - cross: 0x10, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 67b269b330c..e2e91ce99cd 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -88,7 +88,7 @@ module Ci patch ":id/trace.txt" do build = authenticate_build! - error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] content_range = content_range.split('-') @@ -187,14 +187,14 @@ module Ci build = authenticate_build! artifacts_file = build.artifacts_file - unless artifacts_file.file_storage? - return redirect_to build.artifacts_file.url - end - unless artifacts_file.exists? not_found! end + unless artifacts_file.file_storage? + return redirect_to build.artifacts_file.url + end + present_file!(artifacts_file.path, artifacts_file.filename) end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 15a461a16dd..56ad2c77c7d 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,26 +20,26 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, trigger_request = nil) + def jobs_for_ref(ref, tag = false, source = nil) @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, trigger_request) + process?(job[:only], job[:except], ref, tag, source) end end - def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - jobs_for_ref(ref, tag, trigger_request).select do |_, job| + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| job[:stage] == stage end end - def builds_for_ref(ref, tag = false, trigger_request = nil) - jobs_for_ref(ref, tag, trigger_request).map do |name, _| + def builds_for_ref(ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).map do |name, _| build_attributes(name) end end - def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil) - jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _| + def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) end end @@ -50,10 +50,21 @@ module Ci end end + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + def build_attributes(name) job = @jobs[name.to_sym] || {} - { - stage_idx: @stages.index(job[:stage]), + + { stage_idx: @stages.index(job[:stage]), stage: job[:stage], commands: job[:commands], tag_list: job[:tags] || [], @@ -70,9 +81,8 @@ module Ci cache: job[:cache], dependencies: job[:dependencies], after_script: job[:after_script], - environment: job[:environment], - }.compact - } + environment: job[:environment] + }.compact } end def self.validation_message(content) @@ -181,30 +191,35 @@ module Ci end end - def process?(only_params, except_params, ref, tag, trigger_request) + def process?(only_params, except_params, ref, tag, source) if only_params.present? - return false unless matching?(only_params, ref, tag, trigger_request) + return false unless matching?(only_params, ref, tag, source) end if except_params.present? - return false if matching?(except_params, ref, tag, trigger_request) + return false if matching?(except_params, ref, tag, source) end true end - def matching?(patterns, ref, tag, trigger_request) + def matching?(patterns, ref, tag, source) patterns.any? do |pattern| - match_ref?(pattern, ref, tag, trigger_request) + pattern, path = pattern.split('@', 2) + matches_path?(path) && matches_pattern?(pattern, ref, tag, source) end end - def match_ref?(pattern, ref, tag, trigger_request) - pattern, path = pattern.split('@', 2) - return false if path && path != self.path + def matches_path?(path) + return true unless path + + path == self.path + end + + def matches_pattern?(pattern, ref, tag, source) return true if tag && pattern == 'tags' return true if !tag && pattern == 'branches' - return true if trigger_request.present? && pattern == 'triggers' + return true if source_to_pattern(source) == pattern if pattern.first == "/" && pattern.last == "/" Regexp.new(pattern[1...-1]) =~ ref @@ -212,5 +227,13 @@ module Ci pattern == ref end end + + def source_to_pattern(source) + if %w[api external web].include?(source) + source + else + source&.pluralize + end + end end end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 0ea2f97352d..6fc1d56d7a0 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,9 +1,9 @@ class GroupUrlConstrainer def matches?(request) - id = request.params[:group_id] || request.params[:id] + full_path = request.params[:group_id] || request.params[:id] - return false unless DynamicPathValidator.valid_namespace_path?(id) + return false unless DynamicPathValidator.valid_group_path?(full_path) - Group.find_by_full_path(id, follow_redirects: request.get?).present? + Group.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 4444a1abee3..4c0aee6c48f 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -2,7 +2,7 @@ class ProjectUrlConstrainer def matches?(request) namespace_path = request.params[:namespace_id] project_path = request.params[:project_id] || request.params[:id] - full_path = namespace_path + '/' + project_path + full_path = [namespace_path, project_path].join('/') return false unless DynamicPathValidator.valid_project_path?(full_path) diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 28159dc0dec..d16ae7f3f40 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,5 +1,9 @@ class UserUrlConstrainer def matches?(request) - User.find_by_full_path(request.params[:username], follow_redirects: request.get?).present? + full_path = request.params[:username] + + return false unless DynamicPathValidator.valid_user_path?(full_path) + + User.find_by_full_path(full_path, follow_redirects: request.get?).present? end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 7f5f6d9ddb6..c7263f302ab 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -75,10 +75,7 @@ module ContainerRegistry def redirect_response(location) return unless location - # We explicitly remove authorization token - faraday_blob.get(location) do |req| - req['Authorization'] = '' - end + faraday_redirect.get(location) end def faraday @@ -93,5 +90,14 @@ module ContainerRegistry initialize_connection(conn, @options) end end + + # Create a new request to make sure the Authorization header is not inserted + # via the Faraday middleware + def faraday_redirect + @faraday_redirect ||= Faraday.new(@base_uri) do |conn| + conn.request :json + conn.adapter :net_http + end + end end end diff --git a/lib/feature.rb b/lib/feature.rb new file mode 100644 index 00000000000..5650a1c1334 --- /dev/null +++ b/lib/feature.rb @@ -0,0 +1,53 @@ +require 'flipper/adapters/active_record' + +class Feature + # Classes to override flipper table names + class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature + # Using `self.table_name` won't work. ActiveRecord bug? + superclass.table_name = 'features' + end + + class FlipperGate < Flipper::Adapters::ActiveRecord::Gate + superclass.table_name = 'feature_gates' + end + + class << self + def all + flipper.features.to_a + end + + def get(key) + flipper.feature(key) + end + + def persisted?(feature) + # Flipper creates on-memory features when asked for a not-yet-created one. + # If we want to check if a feature has been actually set, we look for it + # on the persisted features list. + all.map(&:name).include?(feature.name) + end + + def enabled?(key) + get(key).enabled? + end + + def enable(key) + get(key).enable + end + + def disable(key) + get(key).disable + end + + private + + def flipper + @flipper ||= begin + adapter = Flipper::Adapters::ActiveRecord.new( + feature_class: FlipperFeature, gate_class: FlipperGate) + + Flipper.new(adapter) + end + end + end +end diff --git a/lib/github/import.rb b/lib/github/import.rb index 06beb607a3e..9c7eb965f93 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,4 +1,5 @@ require_relative 'error' + module Github class Import include Gitlab::ShellAdapter @@ -6,6 +7,7 @@ module Github class MergeRequest < ::MergeRequest self.table_name = 'merge_requests' + self.reset_callbacks :create self.reset_callbacks :save self.reset_callbacks :commit self.reset_callbacks :update @@ -16,6 +18,7 @@ module Github self.table_name = 'issues' self.reset_callbacks :save + self.reset_callbacks :create self.reset_callbacks :commit self.reset_callbacks :update self.reset_callbacks :validate @@ -79,7 +82,7 @@ module Github def fetch_repository begin project.create_repository unless project.repository.exists? - project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.add_remote('github', "https://#{options.fetch(:token)}@github.com/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 8c28009b9c6..4714ab18cc1 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -32,7 +32,7 @@ module Gitlab "Guest" => GUEST, "Reporter" => REPORTER, "Developer" => DEVELOPER, - "Master" => MASTER, + "Master" => MASTER } end @@ -47,7 +47,7 @@ module Gitlab guest: GUEST, reporter: REPORTER, developer: DEVELOPER, - master: MASTER, + master: MASTER } end @@ -60,7 +60,7 @@ module Gitlab "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch." => PROTECTION_DEV_CAN_MERGE, "Partially protected: Developers can push new commits, but cannot force push or delete the branch. Masters can do all of those." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL, + "Fully protected: Developers cannot push new commits, force push, or delete the branch. Only masters can do any of those." => PROTECTION_FULL } end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 96d38f6daa0..3d41ac76406 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -20,21 +20,20 @@ module Gitlab backend: :gitlab_html5, attributes: DEFAULT_ADOC_ATTRS } - context[:pipeline] = :markup + context[:pipeline] = :ascii_doc plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.render(html, context) - html.html_safe end def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = ApplicationSetting.current.plantuml_url - conf.svg_enable = ApplicationSetting.current.plantuml_enabled - conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.url = current_application_settings.plantuml_url + conf.svg_enable = current_application_settings.plantuml_enabled + conf.png_enable = current_application_settings.plantuml_enabled conf.txt_enable = false end end @@ -47,13 +46,13 @@ module Gitlab def stem(node) return super unless node.style.to_sym == :latexmath - %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + %(<pre#{id_attribute(node)} data-math-style="display"><code>#{node.content}</code></pre>) end def inline_quoted(node) return super unless node.type.to_sym == :latexmath - %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + %(<code#{id_attribute(node)} data-math-style="inline">#{node.text}</code>) end private diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index ea918b23a63..da07ba2f2a3 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,8 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) + REGISTRY_SCOPES = [:read_registry].freeze + # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze @@ -11,8 +13,10 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze + AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze + # Other available scopes - OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze + OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) @@ -26,8 +30,8 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password) || oauth_access_token_check(login, password) || - user_with_password_for_git(login, password) || personal_access_token_check(password) || + user_with_password_for_git(login, password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) @@ -37,6 +41,9 @@ module Gitlab end def find_with_user_password(login, password) + # Avoid resource intensive login checks if password is not provided + return unless password.present? + Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) @@ -44,7 +51,7 @@ module Gitlab # LDAP users are only authenticated via LDAP if user.nil? || user.ldap_user? # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? + return unless Gitlab::LDAP::Config.enabled? Gitlab::LDAP::Authentication.login(login, password) else @@ -106,6 +113,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) @@ -118,17 +126,23 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_api_token?(token) - Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_api_token?(token) + token && token.accessible? && valid_scoped_token?(token, ["api"]) end - def valid_api_token?(token) - AccessTokenValidationService.new(token).include_any_scope?(['api']) + def valid_scoped_token?(token, scopes) + AccessTokenValidationService.new(token).include_any_scope?(scopes) + end + + def abilities_for_scope(scopes) + scopes.map do |scope| + self.public_send(:"#{scope}_scope_authentication_abilities") + end.flatten.uniq end def lfs_token_check(login, password) @@ -199,6 +213,16 @@ module Gitlab :create_container_image ] end + alias_method :api_scope_authentication_abilities, :full_authentication_abilities + + def read_registry_scope_authentication_abilities + [:read_container_image] + end + + # The currently used auth method doesn't allow any actions for this scope + def read_user_scope_authentication_abilities + [] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 39b86c61a18..75451cf8aa9 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -15,6 +15,10 @@ module Gitlab def success? actor.present? || type == :ci end + + def failed? + !success? + end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index f34ed0f4cf2..3e0c30c33b7 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -5,7 +5,7 @@ module Gitlab Gitlab::ChatCommands::IssueShow, Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, - Gitlab::ChatCommands::Deploy, + Gitlab::ChatCommands::Deploy ].freeze def execute diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb index 2700a5a2ad5..05994bee79d 100644 --- a/lib/gitlab/chat_commands/presenters/base.rb +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -45,9 +45,9 @@ module Gitlab end def format_response(response) - response[:text] = format(response[:text]) if response.has_key?(:text) + response[:text] = format(response[:text]) if response.key?(:text) - if response.has_key?(:attachments) + if response.key?(:attachments) response[:attachments].each do |attachment| attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] attachment[:text] = format(attachment[:text]) if attachment[:text] diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 8793b20aa35..b6805230348 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,7 +1,20 @@ module Gitlab module Checks class ChangeAccess - # protocol is currently used only in EE + ERROR_MESSAGES = { + push_code: 'You are not allowed to push code to this project.', + delete_default_branch: 'The default branch of a project cannot be deleted.', + force_push_protected_branch: 'You are not allowed to force push code to a protected branch on this project.', + non_master_delete_protected_branch: 'You are not allowed to delete protected branches from this project. Only a project master or owner can delete a protected branch.', + non_web_delete_protected_branch: 'You can only delete protected branches using the web interface.', + merge_protected_branch: 'You are not allowed to merge code into protected branches on this project.', + push_protected_branch: 'You are not allowed to push code to protected branches on this project.', + change_existing_tags: 'You are not allowed to change existing tags on this project.', + update_protected_tag: 'Protected tags cannot be updated.', + delete_protected_tag: 'Protected tags cannot be deleted.', + create_protected_tag: 'You are not allowed to create this tag as it is protected.' + }.freeze + attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( @@ -18,74 +31,87 @@ module Gitlab end def exec - error = push_checks || tag_checks || protected_branch_checks + return true if skip_authorization - if error - GitAccessStatus.new(false, error) - else - GitAccessStatus.new(true) - end + push_checks + branch_checks + tag_checks + + true end protected - def protected_branch_checks - return if skip_authorization + def push_checks + if user_access.cannot_do_action?(:push_code) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end + end + + def branch_checks return unless @branch_name + + if deletion? && @branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end + + protected_branch_checks + end + + def protected_branch_checks return unless ProtectedBranch.protected?(project, @branch_name) if forced_push? - return "You are not allowed to force push code to a protected branch on this project." - elsif deletion? - return "You are not allowed to delete protected branches from this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] end + if deletion? + protected_branch_deletion_checks + else + protected_branch_push_checks + end + end + + def protected_branch_deletion_checks + unless user_access.can_delete_branch?(@branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end + + unless protocol == 'web' + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end + end + + def protected_branch_push_checks if matching_merge_request? - if user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) - return - else - "You are not allowed to merge code into protected branches on this project." + unless user_access.can_merge_to_branch?(@branch_name) || user_access.can_push_to_branch?(@branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] end else - if user_access.can_push_to_branch?(@branch_name) - return - else - "You are not allowed to push code to protected branches on this project." + unless user_access.can_push_to_branch?(@branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_protected_branch] end end end def tag_checks - return if skip_authorization - return unless @tag_name if tag_exists? && user_access.cannot_do_action?(:admin_project) - return "You are not allowed to change existing tags on this project." + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] end protected_tag_checks end def protected_tag_checks - return unless tag_protected? - return "Protected tags cannot be updated." if update? - return "Protected tags cannot be deleted." if deletion? + return unless ProtectedTag.protected?(project, @tag_name) - unless user_access.can_create_tag?(@tag_name) - return "You are not allowed to create this tag as it is protected." - end - end - - def tag_protected? - ProtectedTag.protected?(project, @tag_name) - end + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? - def push_checks - return if skip_authorization - - if user_access.cannot_do_action?(:push_code) - "You are not allowed to push code to this project." + unless user_access.can_create_tag?(@tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] end end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 9b9a0a8125a..a78a85397bd 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -21,7 +21,13 @@ module Gitlab def validate_variables(variables) variables.is_a?(Hash) && - variables.all? { |key, value| validate_string(key) && validate_string(value) } + variables.flatten.all? do |value| + validate_string(value) || validate_integer(value) + end + end + + def validate_integer(value) + value.is_a?(Integer) end def validate_string(value) diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c3b0e651c3a..8acab605c91 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -15,6 +15,10 @@ module Gitlab def self.default {} end + + def value + Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + end end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index dad8c3cdf5b..551483d0aaa 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -11,7 +11,7 @@ module Gitlab def next_time_from(time) @cron_line ||= try_parse_cron(@cron, @cron_timezone) - @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present? + @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present? end def cron_valid? diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb new file mode 100644 index 00000000000..f81f9347b4d --- /dev/null +++ b/lib/gitlab/ci/stage/seed.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + module Stage + class Seed + attr_reader :pipeline + delegate :project, to: :pipeline + + def initialize(pipeline, stage, jobs) + @pipeline = pipeline + @stage = { name: stage } + @jobs = jobs.to_a.dup + end + + def user=(current_user) + @jobs.map! do |attributes| + attributes.merge(user: current_user) + end + end + + def stage + @stage.merge(project: project) + end + + def builds + trigger = pipeline.trigger_requests.first + + @jobs.map do |attributes| + attributes.merge(project: project, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: trigger) + end + end + + def create! + pipeline.stages.create!(stage).tap do |stage| + builds_attributes = builds.map do |attributes| + attributes.merge(stage_id: stage.id) + end + + pipeline.builds.create!(builds_attributes).each do |build| + yield build if block_given? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 57b533bad99..439ef0ce015 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -12,7 +12,7 @@ module Gitlab end def action_path - cancel_namespace_project_build_path(subject.project.namespace, + cancel_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index 3fec2c5d4db..b173c23fba4 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -8,7 +8,7 @@ module Gitlab end def details_path - namespace_project_build_path(subject.project.namespace, + namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index c6139f1b716..e80f3263794 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 505f80848b2..56303e4cb17 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -16,7 +16,7 @@ module Gitlab end def action_path - retry_namespace_project_build_path(subject.project.namespace, + retry_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 0b5199e5483..2778d6f3b52 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -20,7 +20,7 @@ module Gitlab end def action_path - play_namespace_project_build_path(subject.project.namespace, + play_namespace_project_job_path(subject.project.namespace, subject.project, subject) end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index 97c121ce7b9..e5fdc1f8136 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Canceled < Status::Core def text - 'canceled' + s_('CiStatusText|canceled') end def label - 'canceled' + s_('CiStatusLabel|canceled') end def icon diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 0721bf6ec7c..d188bd286a6 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Created < Status::Core def text - 'created' + s_('CiStatusText|created') end def label - 'created' + s_('CiStatusLabel|created') end def icon diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index cb75e9383a8..38e45714c22 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Failed < Status::Core def text - 'failed' + s_('CiStatusText|failed') end def label - 'failed' + s_('CiStatusLabel|failed') end def icon diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index f8f6c2903ba..a4a7edadac9 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Manual < Status::Core def text - 'manual' + s_('CiStatusText|manual') end def label - 'manual action' + s_('CiStatusLabel|manual action') end def icon diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index f40cc1314dc..5164260b861 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Pending < Status::Core def text - 'pending' + s_('CiStatusText|pending') end def label - 'pending' + s_('CiStatusLabel|pending') end def icon diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index 37dfe43fb62..bf7e484ee9b 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -4,11 +4,11 @@ module Gitlab module Pipeline class Blocked < Status::Extended def text - 'blocked' + s_('CiStatusText|blocked') end def label - 'waiting for manual action' + s_('CiStatusLabel|waiting for manual action') end def self.matches?(pipeline, user) diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index 1237cd47dc8..993937e98ca 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Running < Status::Core def text - 'running' + s_('CiStatus|running') end def label - 'running' + s_('CiStatus|running') end def icon diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 28005d91503..0c942920b02 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Skipped < Status::Core def text - 'skipped' + s_('CiStatusText|skipped') end def label - 'skipped' + s_('CiStatusLabel|skipped') end def icon diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 88f7758a270..d7af98857b0 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -3,11 +3,11 @@ module Gitlab module Status class Success < Status::Core def text - 'passed' + s_('CiStatusText|passed') end def label - 'passed' + s_('CiStatusLabel|passed') end def icon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index df6e76b0151..4d7d82e04cf 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -7,11 +7,11 @@ module Gitlab # class SuccessWarning < Status::Extended def text - 'passed' + s_('CiStatusText|passed') end def label - 'passed with warnings' + s_('CiStatusLabel|passed with warnings') end def icon diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fa462cbe095..c4c0623df6c 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -73,7 +73,7 @@ module Gitlab match = "" - stream.each_line do |line| + reverse_line do |line| matches = line.scan(regex) next unless matches.is_a?(Array) next if matches.empty? @@ -86,34 +86,39 @@ module Gitlab nil rescue # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now + # so we just silently ignore error for now end private - def read_last_lines(last_lines) - chunks = [] - pos = lines = 0 - max = stream.size - - # We want an extra line to make sure fist line has full contents - while lines <= last_lines && pos < max - pos += BUFFER_SIZE - - buf = - if pos <= max - stream.seek(-pos, IO::SEEK_END) - stream.read(BUFFER_SIZE) - else # Reached the head, read only left - stream.seek(0) - stream.read(BUFFER_SIZE - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) + def read_last_lines(limit) + to_enum(:reverse_line).first(limit).reverse.join + end + + def reverse_line + stream.seek(0, IO::SEEK_END) + debris = '' + + until (buf = read_backward(BUFFER_SIZE)).empty? + buf += debris + debris, *lines = buf.each_line.to_a + lines.reverse_each do |line| + yield(line.force_encoding('UTF-8')) + end end - chunks.join.lines.last(last_lines).join + yield(debris.force_encoding('UTF-8')) unless debris.empty? + end + + def read_backward(length) + cur_offset = stream.tell + start = cur_offset - length + start = 0 if start < 0 + + stream.seek(start, IO::SEEK_SET) + stream.read(cur_offset - start).tap do + stream.seek(start, IO::SEEK_SET) + end end end end diff --git a/lib/gitlab/ci_access.rb b/lib/gitlab/ci_access.rb new file mode 100644 index 00000000000..def1373d8cf --- /dev/null +++ b/lib/gitlab/ci_access.rb @@ -0,0 +1,9 @@ +module Gitlab + # For backwards compatibility, generic CI (which is a build without a user) is + # allowed to :build_download_code without any other checks. + class CiAccess + def can_do_action?(action) + action == :build_download_code + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 15992b77680..060e013183f 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -28,7 +28,7 @@ module Gitlab union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) - @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 82576d197fe..48735fd197d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -8,45 +8,62 @@ module Gitlab end end - def ensure_application_settings! - return fake_application_settings unless connect_to_db? + delegate :sidekiq_throttling_enabled?, to: :current_application_settings - unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - begin - settings = ::ApplicationSetting.current - # In case Redis isn't running or the Redis UNIX socket file is not available - rescue ::Redis::BaseError, ::Errno::ENOENT - settings = ::ApplicationSetting.last - end + def fake_application_settings + OpenStruct.new(::ApplicationSetting.defaults) + end - settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? + private + + def ensure_application_settings! + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + settings = retrieve_settings_from_database? end settings || in_memory_application_settings end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def retrieve_settings_from_database? + settings = retrieve_settings_from_database_cache? + return settings if settings.present? + + return fake_application_settings unless connect_to_db? + + begin + db_settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + db_settings = ::ApplicationSetting.last + end + db_settings || ::ApplicationSetting.create_from_defaults + end + + def retrieve_settings_from_database_cache? + begin + settings = ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT + # In case Redis isn't running or the Redis UNIX socket file is not available + settings = nil + end + settings + end def in_memory_application_settings @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct fake_application_settings end - def fake_application_settings - OpenStruct.new(::ApplicationSetting.defaults) - end - - private - def connect_to_db? # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + ActiveRecord::Base.connection.table_exists?('application_settings') && + !ActiveRecord::Migrator.needs_migration? rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index bef3b95ff1b..1e11e84a9cb 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -7,7 +7,7 @@ module Gitlab test: :read_build, review: :read_merge_request, staging: :read_build, - production: :read_issue, + production: :read_issue }.freeze def self.get(*args) diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index f78106f5b10..8e74e18a311 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -36,7 +36,7 @@ module Gitlab user: { id: user.try(:id), name: user.try(:name), - email: user.try(:email), + email: user.try(:email) }, commit: { @@ -49,7 +49,7 @@ module Gitlab status: commit.status, duration: commit.duration, started_at: commit.started_at, - finished_at: commit.finished_at, + finished_at: commit.finished_at }, repository: { @@ -60,7 +60,7 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - }, + } } data diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 182a30fd74d..e47fb85b5ee 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages_name, + stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 1ff34553f0a..e81d19a7a2e 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -11,6 +11,7 @@ module Gitlab # ref: String, # user_id: String, # user_name: String, + # user_username: String, # user_email: String # project_id: String, # repository: { @@ -51,6 +52,7 @@ module Gitlab message: message, user_id: user.id, user_name: user.name, + user_username: user.username, user_email: user.email, user_avatar: user.avatar_url, project_id: project.id, diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb new file mode 100644 index 00000000000..b42dc052949 --- /dev/null +++ b/lib/gitlab/data_builder/repository.rb @@ -0,0 +1,35 @@ +module Gitlab + module DataBuilder + module Repository + extend self + + # Produce a hash of post-receive data + def update(project, user, changes, refs) + { + event_name: 'repository_update', + + user_id: user.id, + user_name: user.name, + user_email: user.email, + user_avatar: user.avatar_url, + + project_id: project.id, + project: project.hook_attrs, + + changes: changes, + + refs: refs + } + end + + # Produce a hash of partial data for a single change + def single_change(oldrev, newrev, ref) + { + before: oldrev, + after: newrev, + ref: ref + } + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e76c9abbe04..a412bb6dbd2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -42,7 +42,7 @@ module Gitlab 'in the body of your migration class' end - if Database.postgresql? + if supports_drop_index_concurrently? options = options.merge({ algorithm: :concurrently }) disable_statement_timeout end @@ -50,6 +50,39 @@ module Gitlab remove_index(table_name, options.merge({ column: column_name })) end + # Removes an existing index, concurrently when supported + # + # On PostgreSQL this method removes an index concurrently. + # + # Example: + # + # remove_concurrent_index :users, "index_X_by_Y" + # + # See Rails' `remove_index` for more info on the available arguments. + def remove_concurrent_index_by_name(table_name, index_name, options = {}) + if transaction_open? + raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if supports_drop_index_concurrently? + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout + end + + remove_index(table_name, options.merge({ name: index_name })) + end + + # Only available on Postgresql >= 9.2 + def supports_drop_index_concurrently? + return false unless Database.postgresql? + + version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + + version >= 90200 + end + # Adds a foreign key with only minimal locking on the tables involved. # # This method only requires minimal locking when using PostgreSQL. When diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb new file mode 100644 index 00000000000..3192bf6f667 --- /dev/null +++ b/lib/gitlab/dependency_linker.rb @@ -0,0 +1,27 @@ +module Gitlab + module DependencyLinker + LINKERS = [ + GemfileLinker, + GemspecLinker, + PackageJsonLinker, + ComposerJsonLinker, + PodfileLinker, + PodspecLinker, + PodspecJsonLinker, + CartfileLinker, + GodepsJsonLinker, + RequirementsTxtLinker + ].freeze + + def self.linker(blob_name) + LINKERS.find { |linker| linker.support?(blob_name) } + end + + def self.link(blob_name, plain_text, highlighted_text) + linker = linker(blob_name) + return highlighted_text unless linker + + linker.link(plain_text, highlighted_text) + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb new file mode 100644 index 00000000000..7bbd154eb03 --- /dev/null +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -0,0 +1,86 @@ +module Gitlab + module DependencyLinker + class BaseLinker + URL_REGEX = %r{https?://[^'" ]+}.freeze + REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + + class_attribute :file_type + + def self.support?(blob_name) + Gitlab::FileDetector.type_of(blob_name) == file_type + end + + def self.link(*args) + new(*args).link + end + + attr_accessor :plain_text, :highlighted_text + + def initialize(plain_text, highlighted_text) + @plain_text = plain_text + @highlighted_text = highlighted_text + end + + def link + link_dependencies + + highlighted_lines.join.html_safe + end + + private + + def link_dependencies + raise NotImplementedError + end + + def license_url(name) + Licensee::License.find(name)&.url + end + + def github_url(name) + "https://github.com/#{name}" + end + + def link_tag(name, url) + %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>} + end + + # Links package names based on regex. + # + # Example: + # link_regex(/(github:|:github =>)\s*['"](?<name>[^'"]+)['"]/) + # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` + def link_regex(regex, &url_proc) + highlighted_lines.map!.with_index do |rich_line, i| + marker = StringRegexMarker.new(plain_lines[i], rich_line.html_safe) + + marker.mark(regex, group: :name) do |text, left:, right:| + url = yield(text) + url ? link_tag(text, url) : text + end + end + end + + def plain_lines + @plain_lines ||= plain_text.lines + end + + def highlighted_lines + @highlighted_lines ||= highlighted_text.lines + end + + def regexp_for_value(value, default: /[^'" ]+/) + case value + when Array + Regexp.union(value.map { |v| regexp_for_value(v, default: default) }) + when String + Regexp.escape(value) + when Regexp + value + else + default + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb new file mode 100644 index 00000000000..4f69f2c4ab2 --- /dev/null +++ b/lib/gitlab/dependency_linker/cartfile_linker.rb @@ -0,0 +1,14 @@ +module Gitlab + module DependencyLinker + class CartfileLinker < MethodLinker + self.file_type = :cartfile + + private + + def link_dependencies + link_method_call('github', REPO_REGEX, &method(:github_url)) + link_method_call(%w[github git binary], URL_REGEX, &:itself) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb new file mode 100644 index 00000000000..2fbde7da1b4 --- /dev/null +++ b/lib/gitlab/dependency_linker/cocoapods.rb @@ -0,0 +1,10 @@ +module Gitlab + module DependencyLinker + module Cocoapods + def package_url(name) + package = name.split("/", 2).first + "https://cocoapods.org/pods/#{package}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb new file mode 100644 index 00000000000..0245bf4077a --- /dev/null +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class ComposerJsonLinker < PackageJsonLinker + self.file_type = :composer_json + + private + + def link_packages + link_packages_at_key("require", &method(:package_url)) + link_packages_at_key("require-dev", &method(:package_url)) + end + + def package_url(name) + "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb new file mode 100644 index 00000000000..d034ea67387 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -0,0 +1,32 @@ +module Gitlab + module DependencyLinker + class GemfileLinker < MethodLinker + self.file_type = :gemfile + + private + + def link_dependencies + link_urls + link_packages + end + + def link_urls + # Link `github: "user/repo"` to https://github.com/user/repo + link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) + + # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + + # Link `source "https://rubygems.org"` to https://rubygems.org + link_method_call('source', URL_REGEX, &:itself) + end + + def link_packages + # Link `gem "package_name"` to https://rubygems.org/gems/package_name + link_method_call('gem') do |name| + "https://rubygems.org/gems/#{name}" + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb new file mode 100644 index 00000000000..f1783ee2ab4 --- /dev/null +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -0,0 +1,18 @@ +module Gitlab + module DependencyLinker + class GemspecLinker < MethodLinker + self.file_type = :gemspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + link_method_call('license', &method(:license_url)) + + link_method_call(%w[name add_dependency add_runtime_dependency add_development_dependency]) do |name| + "https://rubygems.org/gems/#{name}" + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb new file mode 100644 index 00000000000..fe091baee6d --- /dev/null +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -0,0 +1,26 @@ +module Gitlab + module DependencyLinker + class GodepsJsonLinker < JsonLinker + NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze + + self.file_type = :godeps_json + + private + + def link_dependencies + link_json('ImportPath') do |path| + case path + when %r{\A(?<repo>gitlab\.com/#{NESTED_REPO_REGEX})\.git/(?<path>.+)\z}, + %r{\A(?<repo>git(lab|hub)\.com/#{REPO_REGEX})/(?<path>.+)\z} + + "https://#{$~[:repo]}/tree/master/#{$~[:path]}" + when /\Agolang\.org/ + "https://godoc.org/#{path}" + else + "https://#{path}" + end + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb new file mode 100644 index 00000000000..a8ef25233d8 --- /dev/null +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class JsonLinker < BaseLinker + def link + return highlighted_text unless json + + super + end + + private + + # Links package names in a JSON key or values. + # + # Example: + # link_json('name') + # # Will link `package` in `"name": "package"` + # + # link_json('name', 'specific_package') + # # Will link `specific_package` in `"name": "specific_package"` + # + # link_json('name', /[^\/]+\/[^\/]+/) + # # Will link `user/repo` in `"name": "user/repo"`, but not `"name": "package"` + # + # link_json('specific_package', '1.0.1', link: :key) + # # Will link `specific_package` in `"specific_package": "1.0.1"` + def link_json(key, value = nil, link: :value, &url_proc) + key = regexp_for_value(key, default: /[^" ]+/) + value = regexp_for_value(value, default: /[^" ]+/) + + if link == :value + value = /(?<name>#{value})/ + else + key = /(?<name>#{key})/ + end + + link_regex(/"#{key}":\s*"#{value}"/, &url_proc) + end + + def json + @json ||= JSON.parse(plain_text) rescue nil + end + end + end +end diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb new file mode 100644 index 00000000000..0ffa2a83c93 --- /dev/null +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -0,0 +1,39 @@ +module Gitlab + module DependencyLinker + class MethodLinker < BaseLinker + private + + # Links package names in a method call or assignment string argument. + # + # Example: + # link_method_call('gem') + # # Will link `package` in `gem "package"`, `gem("package")` and `gem = "package"` + # + # link_method_call('gem', 'specific_package') + # # Will link `specific_package` in `gem "specific_package"` + # + # link_method_call('github', /[^\/"]+\/[^\/"]+/) + # # Will link `user/repo` in `github "user/repo"`, but not `github "package"` + # + # link_method_call(%w[add_dependency add_development_dependency]) + # # Will link `spec.add_dependency "package"` and `spec.add_development_dependency "package"` + # + # link_method_call('name') + # # Will link `package` in `self.name = "package"` + def link_method_call(method_name, value = nil, &url_proc) + method_name = regexp_for_value(method_name) + value = regexp_for_value(value) + + regex = %r{ + #{method_name} # Method name + \s* # Whitespace + [(=]? # Opening brace or equals sign + \s* # Whitespace + ['"](?<name>#{value})['"] # Package name in quotes + }x + + link_regex(regex, &url_proc) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb new file mode 100644 index 00000000000..330c95f0880 --- /dev/null +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -0,0 +1,44 @@ +module Gitlab + module DependencyLinker + class PackageJsonLinker < JsonLinker + self.file_type = :package_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage url], URL_REGEX, &:itself) + + link_packages + end + + def link_packages + link_packages_at_key("dependencies", &method(:package_url)) + link_packages_at_key("devDependencies", &method(:package_url)) + end + + def link_packages_at_key(key, &url_proc) + dependencies = json[key] + return unless dependencies + + dependencies.each do |name, version| + link_json(name, version, link: :key, &url_proc) + + link_json(name) do |value| + case value + when /\A#{URL_REGEX}\z/ + value + when /\A#{REPO_REGEX}\z/ + github_url(value) + end + end + end + end + + def package_url(name) + "https://npmjs.com/package/#{name}" + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb new file mode 100644 index 00000000000..60ad166ea17 --- /dev/null +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -0,0 +1,15 @@ +module Gitlab + module DependencyLinker + class PodfileLinker < GemfileLinker + include Cocoapods + + self.file_type = :podfile + + private + + def link_packages + link_method_call('pod', &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb new file mode 100644 index 00000000000..d82237ed3f1 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb @@ -0,0 +1,32 @@ +module Gitlab + module DependencyLinker + class PodspecJsonLinker < JsonLinker + include Cocoapods + + self.file_type = :podspec_json + + private + + def link_dependencies + link_json('name', json["name"], &method(:package_url)) + link_json('license', &method(:license_url)) + link_json(%w[homepage git], URL_REGEX, &:itself) + + link_packages_at_key("dependencies", &method(:package_url)) + + json["subspecs"]&.each do |subspec| + link_packages_at_key("dependencies", subspec, &method(:package_url)) + end + end + + def link_packages_at_key(key, root = json, &url_proc) + dependencies = root[key] + return unless dependencies + + dependencies.each do |name, _| + link_regex(/"(?<name>#{Regexp.escape(name)})":\s*\[/, &url_proc) + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb new file mode 100644 index 00000000000..a52c7a02439 --- /dev/null +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -0,0 +1,24 @@ +module Gitlab + module DependencyLinker + class PodspecLinker < MethodLinker + include Cocoapods + + STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze + + self.file_type = :podspec + + private + + def link_dependencies + link_method_call('homepage', URL_REGEX, &:itself) + + link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + + link_method_call('license', &method(:license_url)) + link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) + + link_method_call(%w[name dependency], &method(:package_url)) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb new file mode 100644 index 00000000000..2e197e5cd94 --- /dev/null +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -0,0 +1,17 @@ +module Gitlab + module DependencyLinker + class RequirementsTxtLinker < BaseLinker + self.file_type = :requirements_txt + + private + + def link_dependencies + link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=;\[]+)/) do |name| + "https://pypi.python.org/pypi/#{name}" + end + + link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself) + end + end + end +end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 7948782aecc..371cbe04b9b 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -37,6 +37,16 @@ module Gitlab def complete? start_sha && head_sha end + + def compare_in(project) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + project.commit(head_sha) + else + straight = start_sha == base_sha + CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + end + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index c6bf25b5874..2aef7fdaa35 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,16 +1,17 @@ module Gitlab module Diff class File - attr_reader :diff, :repository, :diff_refs + attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs - delegate :new_file, :deleted_file, :renamed_file, - :old_path, :new_path, :a_mode, :b_mode, + delegate :new_file?, :deleted_file?, :renamed_file?, + :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, :submodule?, :too_large?, :collapsed?, to: :diff, prefix: false - def initialize(diff, repository:, diff_refs: nil) + def initialize(diff, repository:, diff_refs: nil, fallback_diff_refs: nil) @diff = diff @repository = repository @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def position(line) @@ -49,24 +50,60 @@ module Gitlab line_code(line) if line end + def old_sha + diff_refs&.base_sha + end + + def new_sha + diff_refs&.head_sha + end + + def content_sha + return old_content_sha if deleted_file? + return @content_sha if defined?(@content_sha) + + refs = diff_refs || fallback_diff_refs + @content_sha = refs&.head_sha + end + def content_commit - return unless diff_refs + return @content_commit if defined?(@content_commit) + + sha = content_sha + @content_commit = repository.commit(sha) if sha + end + + def old_content_sha + return if new_file? + return @old_content_sha if defined?(@old_content_sha) - repository.commit(deleted_file ? old_ref : new_ref) + refs = diff_refs || fallback_diff_refs + @old_content_sha = refs&.base_sha end def old_content_commit - return unless diff_refs + return @old_content_commit if defined?(@old_content_commit) - repository.commit(old_ref) + sha = old_content_sha + @old_content_commit = repository.commit(sha) if sha end - def old_ref - diff_refs.try(:base_sha) + def blob + return @blob if defined?(@blob) + + sha = content_sha + return @blob = nil unless sha + + repository.blob_at(sha, file_path) end - def new_ref - diff_refs.try(:head_sha) + def old_blob + return @old_blob if defined?(@old_blob) + + sha = old_content_sha + return @old_blob = nil unless sha + + @old_blob = repository.blob_at(sha, old_path) end attr_writer :highlighted_diff_lines @@ -85,10 +122,6 @@ module Gitlab @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end - def mode_changed? - a_mode && b_mode && a_mode != b_mode - end - def raw_diff diff.diff.to_s end @@ -117,20 +150,8 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = old_content_commit) - return unless commit - - repository.blob_at(commit.id, old_path) - end - - def blob(commit = content_commit) - return unless commit - - repository.blob_at(commit.id, file_path) - end - def file_identifier - "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}" end end end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 2b9fc65b985..a6007ebf531 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,32 +2,41 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_view, :diff_refs + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs delegate :count, :size, :real_size, to: :diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false) end - def initialize(diffable, project:, diff_options: nil, diff_refs: nil) + def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) diff_options = self.class.default_options.merge(diff_options || {}) - @diffable = diffable - @diffs = diffable.raw_diffs(diff_options) - @project = project + @diffable = diffable + @diffs = diffable.raw_diffs(diff_options) + @project = project @diff_options = diff_options - @diff_refs = diff_refs + @diff_refs = diff_refs + @fallback_diff_refs = fallback_diff_refs end def diff_files @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } end + def diff_file_with_old_path(old_path) + diff_files.find { |diff_file| diff_file.old_path == old_path } + end + + def diff_file_with_new_path(new_path) + diff_files.find { |diff_file| diff_file.new_path == new_path } + end + private def decorate_diff!(diff) - Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs) + Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs) end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 0bd226ef050..9a58b500a2c 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -8,7 +8,8 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs) + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs) end def diff_files diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 7db896522a9..ed2f541977a 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab class Highlight attr_reader :diff_file, :diff_lines, :raw_lines, :repository - delegate :old_path, :new_path, :old_ref, :new_ref, to: :diff_file, prefix: :diff + delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository @@ -61,12 +61,12 @@ module Gitlab def old_lines return unless diff_file - @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_ref, diff_old_path) + @old_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_old_sha, diff_old_path) end def new_lines return unless diff_file - @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_ref, diff_new_path) + @new_lines ||= Gitlab::Highlight.highlight_lines(self.repository, diff_new_sha, diff_new_path) end end end diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb new file mode 100644 index 00000000000..c2a2eb15931 --- /dev/null +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -0,0 +1,17 @@ +module Gitlab + module Diff + class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker + MARKDOWN_SYMBOLS = { + addition: "+", + deletion: "-" + }.freeze + + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + symbol = MARKDOWN_SYMBOLS[mode] + "{#{symbol}#{text}#{symbol}}" + end + end + end + end +end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 736933b1c4b..919965100ae 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,137 +1,21 @@ module Gitlab module Diff - class InlineDiffMarker - MARKDOWN_SYMBOLS = { - addition: "+", - deletion: "-" - }.freeze - - attr_accessor :raw_line, :rich_line - - def initialize(raw_line, rich_line = raw_line) - @raw_line = raw_line - @rich_line = ERB::Util.html_escape(rich_line) - end - - def mark(line_inline_diffs, mode: nil, markdown: false) - return rich_line unless line_inline_diffs - - marker_ranges = [] - line_inline_diffs.each do |inline_diff_range| - # Map the inline-diff range based on the raw line to character positions in the rich line - inline_diff_positions = position_mapping[inline_diff_range].flatten - # Turn the array of character positions into ranges - marker_ranges.concat(collapse_ranges(inline_diff_positions)) - end - - offset = 0 - - # Mark each range - marker_ranges.each_with_index do |range, index| - before_content = - if markdown - "{#{MARKDOWN_SYMBOLS[mode]}" - else - "<span class='#{html_class_names(marker_ranges, mode, index)}'>" - end - after_content = - if markdown - "#{MARKDOWN_SYMBOLS[mode]}}" - else - "</span>" - end - offset = insert_around_range(rich_line, range, before_content, after_content, offset) + class InlineDiffMarker < Gitlab::StringRangeMarker + def mark(line_inline_diffs, mode: nil) + super(line_inline_diffs) do |text, left:, right:| + %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - - rich_line.html_safe end private - def html_class_names(marker_ranges, mode, index) + def html_class_names(left, right, mode) class_names = ["idiff"] - class_names << "left" if index == 0 - class_names << "right" if index == marker_ranges.length - 1 + class_names << "left" if left + class_names << "right" if right class_names << mode if mode class_names.join(" ") end - - # Mapping of character positions in the raw line, to the rich (highlighted) line - def position_mapping - @position_mapping ||= begin - mapping = [] - rich_pos = 0 - (0..raw_line.length).each do |raw_pos| - rich_char = rich_line[rich_pos] - - # The raw and rich lines are the same except for HTML tags, - # so skip over any `<...>` segment - while rich_char == '<' - until rich_char == '>' - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - rich_pos += 1 - rich_char = rich_line[rich_pos] - end - - # multi-char HTML entities in the rich line correspond to a single character in the raw line - if rich_char == '&' - multichar_mapping = [rich_pos] - until rich_char == ';' - rich_pos += 1 - multichar_mapping << rich_pos - rich_char = rich_line[rich_pos] - end - - mapping[raw_pos] = multichar_mapping - else - mapping[raw_pos] = rich_pos - end - - rich_pos += 1 - end - - mapping - end - end - - # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) - return [] if positions.empty? - ranges = [] - - start = prev = positions[0] - range = start..prev - positions[1..-1].each do |pos| - if pos == prev + 1 - range = start..pos - prev = pos - else - ranges << range - start = prev = pos - range = start..prev - end - end - ranges << range - - ranges - end - - # Inserts tags around the characters identified by the given range - def insert_around_range(text, range, before, after, offset = 0) - # Just to be sure - return offset if offset + range.end + 1 > text.length - - text.insert(offset + range.begin, before) - offset += before.length - - text.insert(offset + range.end + 1, after) - offset += after.length - - offset - end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 0a15c6d9358..bd52ae47e9f 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -59,6 +59,10 @@ module Gitlab type == 'match' end + def discussable? + !['match', 'new-nonewline', 'old-nonewline'].include?(type) + end + def as_json(opts = nil) { type: type, diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index fc728123c97..f80afb20f0c 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -12,20 +12,26 @@ module Gitlab attr_reader :head_sha def initialize(attrs = {}) + if diff_file = attrs[:diff_file] + attrs[:diff_refs] = diff_file.diff_refs + attrs[:old_path] = diff_file.old_path + attrs[:new_path] = diff_file.new_path + end + + if diff_refs = attrs[:diff_refs] + attrs[:base_sha] = diff_refs.base_sha + attrs[:start_sha] = diff_refs.start_sha + attrs[:head_sha] = diff_refs.head_sha + end + @old_path = attrs[:old_path] @new_path = attrs[:new_path] + @base_sha = attrs[:base_sha] + @start_sha = attrs[:start_sha] + @head_sha = attrs[:head_sha] + @old_line = attrs[:old_line] @new_line = attrs[:new_line] - - if attrs[:diff_refs] - @base_sha = attrs[:diff_refs].base_sha - @start_sha = attrs[:diff_refs].start_sha - @head_sha = attrs[:diff_refs].head_sha - else - @base_sha = attrs[:base_sha] - @start_sha = attrs[:start_sha] - @head_sha = attrs[:head_sha] - end end # `Gitlab::Diff::Position` objects are stored as serialized attributes in @@ -129,33 +135,19 @@ module Gitlab end def diff_line(repository) - @diff_line ||= diff_file(repository).line_for_position(self) + @diff_line ||= diff_file(repository)&.line_for_position(self) end def line_code(repository) - @line_code ||= diff_file(repository).line_code_for_position(self) + @line_code ||= diff_file(repository)&.line_code_for_position(self) end private def find_diff_file(repository) - # We're at the initial commit, so just get that as we can't compare to anything. - compare = - if Gitlab::Git.blank_ref?(start_sha) - Gitlab::Git::Commit.find(repository.raw_repository, head_sha) - else - Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ) - end - - diff = compare.diffs(paths: paths).first - - return unless diff + return unless diff_refs.complete? - Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index e89ff238ec7..b68a1636814 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -3,21 +3,21 @@ module Gitlab module Diff class PositionTracer - attr_accessor :repository + attr_accessor :project attr_accessor :old_diff_refs attr_accessor :new_diff_refs attr_accessor :paths - def initialize(repository:, old_diff_refs:, new_diff_refs:, paths: nil) - @repository = repository + def initialize(project:, old_diff_refs:, new_diff_refs:, paths: nil) + @project = project @old_diff_refs = old_diff_refs @new_diff_refs = new_diff_refs @paths = paths end - def trace(old_position) + def trace(ab_position) return unless old_diff_refs&.complete? && new_diff_refs&.complete? - return unless old_position.diff_refs == old_diff_refs + return unless ab_position.diff_refs == old_diff_refs # Suppose we have an MR with source branch `feature` and target branch `master`. # When the MR was created, the head of `master` was commit A, and the @@ -44,14 +44,16 @@ module Gitlab # # For diff notes for diff A->B, the position looks like this: # Position - # base_sha - ID of commit A + # start_sha - ID of commit A # head_sha - ID of commit B + # base_sha - ID of base commit of A and B # old_path - path as of A (nil if file was newly created) # new_path - path as of B (nil if file was deleted) # old_line - line number as of A (nil if file was newly created) # new_line - line number as of B (nil if file was deleted) # - # We can easily update `base_sha` and `head_sha` to hold the IDs of commits C and D, + # We can easily update `start_sha` and `head_sha` to hold the IDs of + # commits C and D, and can trivially determine `base_sha` based on those, # but need to find the paths and line numbers as of C and D. # # If the file was unchanged or newly created in A->B, the path as of D can be found @@ -68,107 +70,161 @@ module Gitlab # by generating diff A->C ("base to base"), finding the diff file with # `diff_file.old_path == position.old_path`, and taking `diff_file.new_path`. # The path as of D can be found by taking diff C->D, finding the diff file - # with that same `old_path` and taking `diff_file.new_path`. + # with `old_path` set to that `diff_file.new_path` and taking `diff_file.new_path`. # The line number as of C can be found by using the LineMapper on diff A->C # and providing the line number as of A. # The line number as of D can be found by using the LineMapper on diff C->D # and providing the line number as of C. - results = nil - results ||= trace_added_line(old_position) if old_position.added? || old_position.unchanged? - results ||= trace_removed_line(old_position) if old_position.removed? || old_position.unchanged? - - return unless results - - file_diff, old_line, new_line = results - - new_position = Position.new( - old_path: file_diff.old_path, - new_path: file_diff.new_path, - head_sha: new_diff_refs.head_sha, - start_sha: new_diff_refs.start_sha, - base_sha: new_diff_refs.base_sha, - old_line: old_line, - new_line: new_line - ) - - # If a position is found, but is not actually contained in the diff, for example - # because it was an unchanged line in the context of a change that was undone, - # we cannot return this as a successful trace. - return unless new_position.diff_line(repository) - - new_position + if ab_position.added? + trace_added_line(ab_position) + elsif ab_position.removed? + trace_removed_line(ab_position) + else # unchanged + trace_unchanged_line(ab_position) + end end private - def trace_added_line(old_position) - file_path = old_position.new_path - - return unless diff_head_to_head - - file_head_to_head = diff_head_to_head.find { |diff_file| diff_file.old_path == file_path } - - file_path = file_head_to_head.new_path if file_head_to_head - - new_line = LineMapper.new(file_head_to_head).old_to_new(old_position.new_line) - - return unless new_line - - file_diff = new_diffs.find { |diff_file| diff_file.new_path == file_path } - return unless file_diff - - old_line = LineMapper.new(file_diff).new_to_old(new_line) - - [file_diff, old_line, new_line] + def trace_added_line(ab_position) + b_path = ab_position.new_path + b_line = ab_position.new_line + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_path = bd_diff&.new_path || b_path + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + if d_line + cd_diff = cd_diffs.diff_file_with_new_path(d_path) + + c_path = cd_diff&.old_path || d_path + c_line = LineMapper.new(cd_diff).new_to_old(d_line) + + if c_line + # If the line is still in D but also in C, it has turned from an + # added line into an unchanged one. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff, so we treat it as outdated. + ac_diff = ac_diffs.diff_file_with_new_path(c_path) + + { position: position(ac_diff, nil, c_line), outdated: true } + end + else + # If the line is still in D and not in C, it is still added. + { position: position(cd_diff, nil, d_line), outdated: false } + end + else + # If the line is no longer in D, it has been removed from the MR. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def trace_removed_line(old_position) - file_path = old_position.old_path + def trace_removed_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line - return unless diff_base_to_base + ac_diff = ac_diffs.diff_file_with_old_path(a_path) - file_base_to_base = diff_base_to_base.find { |diff_file| diff_file.old_path == file_path } + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) - file_path = file_base_to_base.old_path if file_base_to_base + if c_line + cd_diff = cd_diffs.diff_file_with_old_path(c_path) - old_line = LineMapper.new(file_base_to_base).old_to_new(old_position.old_line) + d_path = cd_diff&.new_path || c_path + d_line = LineMapper.new(cd_diff).old_to_new(c_line) - return unless old_line + if d_line + # If the line is still in C but also in D, it has turned from a + # removed line into an unchanged one. + bd_diff = bd_diffs.diff_file_with_new_path(d_path) - file_diff = new_diffs.find { |diff_file| diff_file.old_path == file_path } - return unless file_diff - - new_line = LineMapper.new(file_diff).old_to_new(old_line) + { position: position(bd_diff, nil, d_line), outdated: true } + else + # If the line is still in C and not in D, it is still removed. + { position: position(cd_diff, c_line, nil), outdated: false } + end + else + # If the line is no longer in C, it has been removed outside of the MR. + { position: position(ac_diff, a_line, nil), outdated: true } + end + end - [file_diff, old_line, new_line] + def trace_unchanged_line(ab_position) + a_path = ab_position.old_path + a_line = ab_position.old_line + b_path = ab_position.new_path + b_line = ab_position.new_line + + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + c_path = ac_diff&.new_path || a_path + c_line = LineMapper.new(ac_diff).old_to_new(a_line) + + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + d_line = LineMapper.new(bd_diff).old_to_new(b_line) + + cd_diff = cd_diffs.diff_file_with_old_path(c_path) + + if c_line && d_line + # If the line is still in C and D, it is still unchanged. + new_position = position(cd_diff, c_line, d_line) + if valid_position?(new_position) + # If the line is still in the MR, we don't treat this as outdated. + { position: new_position, outdated: false } + else + # If the line is no longer in the MR, we unfortunately cannot show + # the current state on the CD diff or any change on the BD diff, + # so we treat it as outdated. + { position: nil, outdated: true } + end + elsif d_line # && !c_line + # If the line is still in D but no longer in C, it has turned from + # an unchanged line into an added one. + # We don't treat this as outdated since the line is still in the MR. + { position: position(cd_diff, nil, d_line), outdated: false } + else # !d_line && (c_line || !c_line) + # If the line is no longer in D, it has turned from an unchanged line + # into a removed one. + { position: position(bd_diff, b_line, nil), outdated: true } + end end - def diff_base_to_base - @diff_base_to_base ||= diff_files(old_diff_refs.base_sha || old_diff_refs.start_sha, new_diff_refs.base_sha || new_diff_refs.start_sha) + def ac_diffs + @ac_diffs ||= compare( + old_diff_refs.base_sha || old_diff_refs.start_sha, + new_diff_refs.base_sha || new_diff_refs.start_sha, + straight: true + ) end - def diff_head_to_head - @diff_head_to_head ||= diff_files(old_diff_refs.head_sha, new_diff_refs.head_sha) + def bd_diffs + @bd_diffs ||= compare(old_diff_refs.head_sha, new_diff_refs.head_sha, straight: true) end - def new_diffs - @new_diffs ||= diff_files(new_diff_refs.start_sha, new_diff_refs.head_sha, use_base: true) + def cd_diffs + @cd_diffs ||= compare(new_diff_refs.start_sha, new_diff_refs.head_sha) end - def diff_files(start_sha, head_sha, use_base: false) - base_sha = self.repository.merge_base(start_sha, head_sha) || start_sha + def compare(start_sha, head_sha, straight: false) + compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + compare.diffs(paths: paths, expanded: true) + end - diffs = self.repository.raw_repository.diff( - use_base ? base_sha : start_sha, - head_sha, - {}, - *paths - ) + def position(diff_file, old_line, new_line) + Position.new(diff_file: diff_file, old_line: old_line, new_line: new_line) + end - diffs.decorate! do |diff| - Gitlab::Diff::File.new(diff, repository: self.repository) - end + def valid_position?(position) + !!position.diff_line(project.repository) end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 496ee0bdcb0..38e27513281 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -131,10 +131,12 @@ module Gitlab def check_patch(patch_path) step("Checking out master", %w[git checkout master]) step("Resetting to latest master", %w[git reset --hard origin/master]) + step("Fetching CE/#{ce_branch}", %W[git fetch #{CE_REPO} #{ce_branch}]) step( "Checking if #{patch_path} applies cleanly to EE/master", %W[git apply --check --3way #{patch_path}] ) do |output, status| + puts output unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') @@ -310,6 +312,17 @@ module Gitlab Resolve them, stage the changes and commit them. + If the patch couldn't be applied cleanly, use the following command: + + # In the EE repo + $ git apply --reject path/to/#{ce_branch}.patch + + This option makes git apply the parts of the patch that are applicable, + and leave the rejected hunks in corresponding `.rej` files. + You can then resolve the conflicts highlighted in `.rej` by + manually applying the correct diff from the `.rej` file to the file with conflicts. + When finished, you can delete the `.rej` files and commit your changes. + ⚠️ Don't forget to push your branch to gitlab-ee: # In the EE repo diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 6c69cd9e6a9..ea035e33eff 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -42,7 +42,7 @@ module Gitlab return unless compare # This diff is more moderated in number of files and lines - @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files + @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files end def diffs_count diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb new file mode 100644 index 00000000000..781f9c56a42 --- /dev/null +++ b/lib/gitlab/encoding_helper.rb @@ -0,0 +1,62 @@ +module Gitlab + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect && detect[:encoding] + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end +end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 270d67dd50c..7f884183bb1 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -6,12 +6,13 @@ module Gitlab end def call(env) - route = Gitlab::EtagCaching::Router.match(env) + request = Rack::Request.new(env) + route = Gitlab::EtagCaching::Router.match(request) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(env) + etag, cached_value_present = get_etag(request) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -27,8 +28,8 @@ module Gitlab private - def get_etag(env) - cache_key = env['PATH_INFO'] + def get_etag(request) + cache_key = request.path store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index d74e31af5c6..dccc66b3918 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -7,18 +7,20 @@ module Gitlab # - Don't contain a reserved word (expect for the words used in the # regex itself) # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route - # - Ending in `issues/id`/rendered_title` for the `issue_title` route - USED_IN_ROUTES = %w[noteable issue notes issues rendered_title - commit pipelines merge_requests new].freeze - RESERVED_WORDS = Gitlab::Regex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES - RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + commit pipelines merge_requests builds + new environments].freeze + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) + ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/realtime_changes\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( @@ -41,10 +43,18 @@ module Gitlab %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z), 'project_pipeline' ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z), + 'project_build' + ), + Gitlab::EtagCaching::Router::Route.new( + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), + 'environments' + ) ].freeze - def self.match(env) - ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + def self.match(request) + ROUTES.find { |route| route.regexp.match(request.path_info) } end end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index c9ca4cadd1c..a8cb7fc3fe7 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -5,15 +5,33 @@ module Gitlab # a README or a CONTRIBUTING file. module FileDetector PATTERNS = { + # Project files readme: /\Areadme/i, changelog: /\A(changelog|history|changes|news)/i, license: /\A(licen[sc]e|copying)(\..+|\z)/i, contributing: /\Acontributing/i, version: 'version', + avatar: /\Alogo\.(png|jpg|gif)\z/, + + # Configuration files gitignore: '.gitignore', koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', - avatar: /\Alogo\.(png|jpg|gif)\z/ + route_map: 'route-map.yml', + + # Dependency files + cartfile: /\ACartfile/, + composer_json: 'composer.json', + gemfile: /\A(Gemfile|gems\.rb)\z/, + gemfile_lock: 'Gemfile.lock', + gemspec: /\.gemspec\z/, + godeps_json: 'Godeps.json', + package_json: 'package.json', + podfile: 'Podfile', + podspec_json: /\.podspec\.json\z/, + podspec: /\.podspec\z/, + requirements_txt: /requirements\.txt\z/, + yarn_lock: 'yarn.lock' }.freeze # Returns an Array of file types based on the given paths. diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb new file mode 100644 index 00000000000..093d9ed8092 --- /dev/null +++ b/lib/gitlab/file_finder.rb @@ -0,0 +1,32 @@ +# This class finds files in a repository by name and content +# the result is joined and sorted by file name +module Gitlab + class FileFinder + BATCH_SIZE = 100 + + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + end + + def find(query) + blobs = project.repository.search_files_by_content(query, ref).first(BATCH_SIZE) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = Gitlab::ProjectSearchResults.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, ref).first(BATCH_SIZE).each do |filename| + results << [filename, OpenStruct.new(ref: ref)] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 58193391926..66829a03c2e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Blame - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_reader :lines, :blames diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 12458f9f410..d60e607b02b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Blob include Linguist::BlobHelper - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection @@ -88,9 +88,10 @@ module Gitlab new( id: blob_entry[:oid], name: blob_entry[:name], + size: 0, data: '', path: path, - commit_id: sha, + commit_id: sha ) end end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 586380da94a..124526e4b59 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,40 @@ module Gitlab module Git class Branch < Ref + def initialize(repository, name, target) + if target.is_a?(Gitaly::FindLocalBranchResponse) + target = target_from_gitaly_local_branches_response(target) + end + + super(repository, name, target) + end + + def target_from_gitaly_local_branches_response(response) + # Git messages have no encoding enforcements. However, in the UI we only + # handle UTF-8, so basically we cross our fingers that the message force + # encoded to UTF-8 is readable. + message = response.commit_subject.dup.force_encoding('UTF-8') + + # NOTE: For ease of parsing in Gitaly, we have only the subject of + # the commit and not the full message. This is ok, since all the + # code that uses `local_branches` only cares at most about the + # commit message. + # TODO: Once gitaly "takes over" Rugged consider separating the + # subject from the message to make it clearer when there's one + # available but not the other. + hash = { + id: response.commit_id, + message: message, + authored_date: Time.at(response.commit_author.date.seconds), + author_name: response.commit_author.name, + author_email: response.commit_author.email, + committed_date: Time.at(response.commit_committer.date.seconds), + committer_name: response.commit_committer.name, + committer_email: response.commit_committer.email + } + + Gitlab::Git::Commit.decorate(hash) + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 3a73697dc5d..bb04731f08c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Commit - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :raw_commit, :head, :refs @@ -19,13 +19,7 @@ module Gitlab def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) - methods = [:message, :parent_ids, :authored_date, :author_name, - :author_email, :committed_date, :committer_name, - :committer_email] - - methods.all? do |method| - send(method) == other.send(method) - end + id && id == other.id end class << self @@ -55,6 +49,7 @@ module Gitlab # Commit.find(repo, 'master') # def find(repo, commit_id = "HEAD") + return commit_id if commit_id.is_a?(Gitlab::Git::Commit) return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) obj = if commit_id.is_a?(String) @@ -192,6 +187,10 @@ module Gitlab Commit.diff_from_parent(raw_commit, options) end + def deltas + @deltas ||= diff_from_parent.each_delta.map { |d| Gitlab::Git::Diff.new(d) } + end + def has_zero_stats? stats.total.zero? rescue diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 696a2acd5e3..78e440395a5 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -3,7 +3,7 @@ module Gitlab class Compare attr_reader :head, :base, :straight - def initialize(repository, base, head, straight = false) + def initialize(repository, base, head, straight: false) @repository = repository @straight = straight diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 019be151353..8926aa19925 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -3,7 +3,7 @@ module Gitlab module Git class Diff TimeoutError = Class.new(StandardError) - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Diff properties attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff @@ -11,15 +11,34 @@ module Gitlab # Stats properties attr_accessor :new_file, :renamed_file, :deleted_file - attr_accessor :too_large + alias_method :new_file?, :new_file + alias_method :deleted_file?, :deleted_file + alias_method :renamed_file?, :renamed_file - # The maximum size of a diff to display. - DIFF_SIZE_LIMIT = 102400 # 100 KB + attr_accessor :expanded - # The maximum size before a diff is collapsed. - DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + # We need this accessor because of `to_hash` and `init_from_hash` + attr_accessor :too_large class << self + # The maximum size of a diff to display. + def size_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 200.kilobytes + else + 100.kilobytes + end + end + + # The maximum size before a diff is collapsed. + def collapse_limit + if Feature.enabled?('gitlab_git_diff_size_limit_increase') + 100.kilobytes + else + 10.kilobytes + end + end + def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -148,7 +167,7 @@ module Gitlab :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, :ignore_filemode, :recurse_ignored_dirs, :paths, - :max_files, :max_lines, :all_diffs, :no_collapse] + :max_files, :max_lines, :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -173,16 +192,20 @@ module Gitlab end end - def initialize(raw_diff, collapse: false) + def initialize(raw_diff, expanded: true) + @expanded = expanded + case raw_diff when Hash init_from_hash(raw_diff) - prune_diff_if_eligible(collapse) + prune_diff_if_eligible when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff, collapse: collapse) - when Gitaly::CommitDiffResponse + init_from_rugged(raw_diff) + when Gitlab::GitalyClient::Diff + init_from_gitaly(raw_diff) + prune_diff_if_eligible + when Gitaly::CommitDelta init_from_gitaly(raw_diff) - prune_diff_if_eligible(collapse) when nil raise "Nil as raw diff passed" else @@ -206,6 +229,10 @@ module Gitlab hash end + def mode_changed? + a_mode && b_mode && a_mode != b_mode + end + def submodule? a_mode == '160000' || b_mode == '160000' end @@ -216,17 +243,13 @@ module Gitlab def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + @too_large = @diff.bytesize >= self.class.size_limit else @too_large end end - def collapsible? - @diff.bytesize >= DIFF_COLLAPSE_LIMIT - end - - def prune_large_diff! + def too_large! @diff = '' @line_count = 0 @too_large = true @@ -234,10 +257,11 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - false + + @collapsed = !expanded && @diff.bytesize >= self.class.collapse_limit end - def prune_collapsed_diff! + def collapse! @diff = '' @line_count = 0 @collapsed = true @@ -245,9 +269,9 @@ module Gitlab private - def init_from_rugged(rugged, collapse: false) + def init_from_rugged(rugged) if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged, collapse: collapse) + init_from_rugged_patch(rugged) d = rugged.delta else d = rugged @@ -262,10 +286,10 @@ module Gitlab @deleted_file = d.deleted? end - def init_from_rugged_patch(patch, collapse: false) + def init_from_rugged_patch(patch) # Don't bother initializing diffs that are too large. If a diff is # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch, collapse) + return if !patch.delta.binary? && prune_large_patch(patch) @diff = encode!(strip_diff_headers(patch.to_s)) end @@ -278,40 +302,43 @@ module Gitlab end end - def init_from_gitaly(diff_msg) - @diff = diff_msg.raw_chunks.join - @new_path = encode!(diff_msg.to_path.dup) - @old_path = encode!(diff_msg.from_path.dup) - @a_mode = diff_msg.old_mode.to_s(8) - @b_mode = diff_msg.new_mode.to_s(8) - @new_file = diff_msg.from_id == BLANK_SHA - @renamed_file = diff_msg.from_path != diff_msg.to_path - @deleted_file = diff_msg.to_id == BLANK_SHA + def init_from_gitaly(diff) + @diff = diff.patch if diff.respond_to?(:patch) + @new_path = encode!(diff.to_path.dup) + @old_path = encode!(diff.from_path.dup) + @a_mode = diff.old_mode.to_s(8) + @b_mode = diff.new_mode.to_s(8) + @new_file = diff.from_id == BLANK_SHA + @renamed_file = diff.from_path != diff.to_path + @deleted_file = diff.to_id == BLANK_SHA end - def prune_diff_if_eligible(collapse = false) - prune_large_diff! if too_large? - prune_collapsed_diff! if collapse && collapsible? + def prune_diff_if_eligible + if too_large? + too_large! + elsif collapsed? + collapse! + end end # If the patch surpasses any of the diff limits it calls the appropiate # prune method and returns true. Otherwise returns false. - def prune_large_patch(patch, collapse) + def prune_large_patch(patch) size = 0 patch.each_hunk do |hunk| hunk.each_line do |line| size += line.content.bytesize - if size >= DIFF_SIZE_LIMIT - prune_large_diff! + if size >= self.class.size_limit + too_large! return true end end end - if collapse && size >= DIFF_COLLAPSE_LIMIT - prune_collapsed_diff! + if !expanded && size >= self.class.collapse_limit + collapse! return true end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 4e45ec7c174..334e06a6eca 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,35 +9,29 @@ module Gitlab @iterator = iterator @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) - @max_bytes = @max_files * 5120 # Average 5 KB per file + @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min - @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file - @all_diffs = !!options.fetch(:all_diffs, false) - @no_collapse = !!options.fetch(:no_collapse, true) - @deltas_only = !!options.fetch(:deltas_only, false) + @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file + @enforce_limits = !!options.fetch(:limits, true) + @expanded = !!options.fetch(:expanded, true) @line_count = 0 @byte_count = 0 @overflow = false + @empty = true @array = Array.new end def each(&block) - if @populated - # @iterator.each is slower than just iterating the array in place - @array.each(&block) - elsif @deltas_only - each_delta(&block) - else - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do - each_patch(&block) - end + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do + each_patch(&block) end end def empty? - !@iterator.any? + any? # Make sure the iterator has been exercised + @empty end def overflow? @@ -63,17 +57,17 @@ module Gitlab collection = each_with_index do |element, i| @array[i] = yield(element) end - @populated = true collection end + alias_method :to_ary, :to_a + private def populate! return if @populated each { nil } # force a loop through all diffs - @populated = true nil end @@ -81,42 +75,36 @@ module Gitlab files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes end - def each_delta - @iterator.each_delta.with_index do |delta, i| - diff = Gitlab::Git::Diff.new(delta) - - yield @array[i] = diff + def each_patch + i = 0 + @array.each do |diff| + yield diff + i += 1 end - end - def each_patch - @iterator.each_with_index do |raw, i| - # First yield cached Diff instances from @array - if @array[i] - yield @array[i] - next - end + return if @overflow + return if @iterator.nil? - # We have exhausted @array, time to create new Diff instances or stop. - break if @overflow + @iterator.each do |raw| + @empty = false - if !@all_diffs && i >= @max_files + if @enforce_limits && i >= @max_files @overflow = true break end - collapse = !@all_diffs && !@no_collapse + expanded = !@enforce_limits || @expanded - diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + diff = Gitlab::Git::Diff.new(raw, expanded: expanded) - if collapse && over_safe_limits?(i) - diff.prune_collapsed_diff! + if !expanded && over_safe_limits?(i) + diff.collapse! end @line_count += diff.line_count @byte_count += diff.diff.bytesize - if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes) # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -124,7 +112,13 @@ module Gitlab end yield @array[i] = diff + i += 1 end + + @populated = true + + # Allow iterator to be garbage-collected. It cannot be reused anyway. + @iterator = nil end end end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb deleted file mode 100644 index f918074cb14..00000000000 --- a/lib/gitlab/git/encoding_helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -module Gitlab - module Git - module EncodingHelper - extend self - - # This threshold is carefully tweaked to prevent usage of encodings detected - # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, - # we're better off sticking with utf8 encoding. - # Reason: git diff can return strings with invalid utf8 byte sequences if it - # truncates a diff in the middle of a multibyte character. In this case - # CharlockHolmes will try to guess the encoding and will likely suggest an - # obscure encoding with low confidence. - # There is a lot more info with this merge request: - # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 - ENCODING_CONFIDENCE_THRESHOLD = 40 - - def encode!(message) - return nil unless message.respond_to? :force_encoding - - # if message is utf-8 encoding, just return it - message.force_encoding("UTF-8") - return message if message.valid_encoding? - - # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) - return message.force_encoding("BINARY") if detect && detect[:type] == :binary - - # force detected encoding if we have sufficient confidence. - if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD - message.force_encoding(detect[:encoding]) - end - - # encode and clean the bad chars - message.replace clean(message) - rescue - encoding = detect ? detect[:encoding] : "unknown" - "--broken encoding: #{encoding}" - end - - def encode_utf8(message) - detect = CharlockHolmes::EncodingDetector.detect(message) - if detect - begin - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') - rescue ArgumentError => e - Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") - - '' - end - else - clean(message) - end - end - - private - - def clean(message) - message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") - .encode("UTF-8") - .gsub("\0".encode("UTF-8"), "") - end - end - end -end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 37ef6836742..ebf7393dc61 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Ref - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 239dc663598..9d6adbdb4ac 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -80,14 +80,16 @@ module Gitlab end # Returns an Array of Branches - def branches - rugged.branches.map do |rugged_ref| + def branches(filter: nil, sort_by: nil) + branches = rugged.branches.each(filter).map do |rugged_ref| begin Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) rescue Rugged::ReferenceError # Omit invalid branch end - end.compact.sort_by(&:name) + end.compact + + sort_branches(branches, sort_by) end def reload_rugged @@ -108,15 +110,21 @@ module Gitlab Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref end - def local_branches - rugged.branches.each(:local).map do |branch| - Gitlab::Git::Branch.new(self, branch.name, branch.target) + def local_branches(sort_by: nil) + gitaly_migrate(:local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.local_branches(sort_by: sort_by).map do |gitaly_branch| + Gitlab::Git::Branch.new(self, gitaly_branch.name, gitaly_branch) + end + else + branches(filter: :local, sort_by: sort_by) + end end end # Returns the number of valid branches def branch_count - Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + gitaly_migrate(:branch_names) do |is_enabled| if is_enabled gitaly_ref_client.count_branch_names else @@ -135,7 +143,7 @@ module Gitlab # Returns the number of valid tags def tag_count - Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + gitaly_migrate(:tag_names) do |is_enabled| if is_enabled gitaly_ref_client.count_tag_names else @@ -260,7 +268,7 @@ module Gitlab 'RepoPath' => path, 'ArchivePrefix' => prefix, 'ArchivePath' => archive_file_path(prefix, storage_path, format), - 'CommitId' => commit.id, + 'CommitId' => commit.id } end @@ -998,31 +1006,39 @@ module Gitlab # Parses the contents of a .gitmodules file and returns a hash of # submodule information. def parse_gitmodules(commit, content) - results = {} + modules = {} - current = "" - content.split("\n").each do |txt| - if txt =~ /^\s*\[/ - current = txt.match(/(?<=").*(?=")/)[0] - results[current] = {} - else - next unless results[current] - match_data = txt.match(/(\w+)\s*=\s*(.*)/) - next unless match_data - target = match_data[2].chomp - results[current][match_data[1]] = target + name = nil + content.each_line do |line| + case line.strip + when /\A\[submodule "(?<name>[^"]+)"\]\z/ # Submodule header + name = $~[:name] + modules[name] = {} + when /\A(?<key>\w+)\s*=\s*(?<value>.*)\z/ # Key/value pair + key = $~[:key] + value = $~[:value].chomp + + next unless name && modules[name] - if match_data[1] == "path" + modules[name][key] = value + + if key == 'path' begin - results[current]["id"] = blob_content(commit, target) + modules[name]['id'] = blob_content(commit, value) rescue InvalidBlobName - results.delete(current) + # The current entry is invalid + modules.delete(name) + name = nil end end + when /\A#/ # Comment + next + else # Invalid line + name = nil end end - results + modules end # Returns true if +commit+ introduced changes to +path+, using commit @@ -1078,7 +1094,12 @@ module Gitlab elsif tmp_entry.nil? return nil else - tmp_entry = rugged.lookup(tmp_entry[:oid]) + begin + tmp_entry = rugged.lookup(tmp_entry[:oid]) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + return nil + end + return nil unless tmp_entry.type == :tree tmp_entry = tmp_entry[dir] end @@ -1110,56 +1131,6 @@ module Gitlab end end - def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) - git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) - - # Put files into a directory before archiving - prefix = "#{archive_name(treeish)}/" - git_archive_cmd << "--prefix=#{prefix}" - - # Format defaults to tar - git_archive_cmd << "--format=#{format}" if format - - git_archive_cmd += %W(-- #{treeish}) - - open(filename, 'w') do |file| - # Create a pipe to act as the '|' in 'git archive ... | gzip' - pipe_rd, pipe_wr = IO.pipe - - # Get the compression process ready to accept data from the read end - # of the pipe - compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) - # The read end belongs to the compression process now; we should - # close our file descriptor for it. - pipe_rd.close - - # Start 'git archive' and tell it to write into the write end of the - # pipe. - git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) - # The write end belongs to 'git archive' now; close it. - pipe_wr.close - - # When 'git archive' and the compression process are finished, we are - # done. - Process.waitpid(git_archive_pid) - raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? - Process.waitpid(compress_pid) - raise "#{compress_cmd.join(' ')} failed" unless $?.success? - end - end - - def nice(cmd) - nice_cmd = %w(nice -n 20) - unless unsupported_platform? - nice_cmd += %w(ionice -c 2 -n 7) - end - nice_cmd + cmd - end - - def unsupported_platform? - %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? - end - # Returns true if the index entry has the special file mode that denotes # a submodule. def submodule?(index_entry) @@ -1252,6 +1223,23 @@ module Gitlab diff.each_patch end + def sort_branches(branches, sort_by) + case sort_by + when 'name' + branches.sort_by(&:name) + when 'updated_desc' + branches.sort do |a, b| + b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date + end + when 'updated_asc' + branches.sort do |a, b| + a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date + end + else + branches + end + end + def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b722d8a9f56..b9afa05c819 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,7 +1,7 @@ module Gitlab module Git class Tree - include Gitlab::Git::EncodingHelper + include Gitlab::EncodingHelper attr_accessor :id, :root_id, :name, :path, :type, :mode, :commit_id, :submodule_url @@ -35,7 +35,7 @@ module Gitlab type: entry[:type], mode: entry[:filemode].to_s(8), path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha, + commit_id: sha ) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 99724db8da2..0a19d24eb20 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -3,33 +3,39 @@ module Gitlab class GitAccess UnauthorizedError = Class.new(StandardError) + NotFoundError = Class.new(StandardError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', deploy_key_upload: 'This deploy key does not have write access to this project.', - no_repo: 'A repository for this project does not exist yet.' + no_repo: 'A repository for this project does not exist yet.', + project_not_found: 'The project you were looking for could not be found.', + account_blocked: 'Your account has been blocked.', + command_not_allowed: "The command you're trying to execute is not allowed.", + upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', + receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.' }.freeze DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities + attr_reader :actor, :project, :protocol, :authentication_abilities def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities - @user_access = UserAccess.new(user, project: project) end def check(cmd, changes) check_protocol! check_active_user! check_project_accessibility! + check_command_disabled!(cmd) check_command_existence!(cmd) check_repository_existence! @@ -40,9 +46,7 @@ module Gitlab check_push_access!(changes) end - build_status_object(true) - rescue UnauthorizedError => ex - build_status_object(false, ex.message) + true end def guest_can_download_code? @@ -73,19 +77,39 @@ module Gitlab return if deploy_key? if user && !user_access.allowed? - raise UnauthorizedError, "Your account has been blocked." + raise UnauthorizedError, ERROR_MESSAGES[:account_blocked] end end def check_project_accessibility! if project.blank? || !can_read_project? - raise UnauthorizedError, 'The project you were looking for could not be found.' + raise NotFoundError, ERROR_MESSAGES[:project_not_found] + end + end + + def check_command_disabled!(cmd) + if upload_pack?(cmd) + check_upload_pack_disabled! + elsif receive_pack?(cmd) + check_receive_pack_disabled! + end + end + + def check_upload_pack_disabled! + if http? && upload_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http] + end + end + + def check_receive_pack_disabled! + if http? && receive_pack_disabled_over_http? + raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http] end end def check_command_existence!(cmd) unless ALL_COMMANDS.include?(cmd) - raise UnauthorizedError, "The command you're trying to execute is not allowed." + raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed] end end @@ -138,11 +162,9 @@ module Gitlab # Iterate over all changes to find if user allowed all of them to be applied changes_list.each do |change| - status = check_single_change_access(change) - unless status.allowed? - # If user does not have access to make at least one change - cancel all push - raise UnauthorizedError, status.message - end + # If user does not have access to make at least one change, cancel all + # push by allowing the exception to bubble up + check_single_change_access(change) end end @@ -168,14 +190,40 @@ module Gitlab actor.is_a?(DeployKey) end + def ci? + actor == :ci + end + def can_read_project? - if deploy_key + if deploy_key? deploy_key.has_access_to?(project) elsif user user.can?(:read_project, project) + elsif ci? + true # allow CI (build without a user) for backwards compatibility end || Guest.can?(:read_project, project) end + def http? + protocol == 'http' + end + + def upload_pack?(command) + command == 'git-upload-pack' + end + + def receive_pack?(command) + command == 'git-receive-pack' + end + + def upload_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.upload_pack + end + + def receive_pack_disabled_over_http? + !Gitlab.config.gitlab_shell.receive_pack + end + protected def user @@ -185,15 +233,19 @@ module Gitlab case actor when User actor - when DeployKey - nil when Key - actor.user + actor.user unless actor.is_a?(DeployKey) + when :ci + nil end end - def build_status_object(status, message = '') - Gitlab::GitAccessStatus.new(status, message) + def user_access + @user_access ||= if ci? + CiAccess.new + else + UserAccess.new(user, project: project) + end end end end diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb deleted file mode 100644 index 09bb01be694..00000000000 --- a/lib/gitlab/git_access_status.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Gitlab - class GitAccessStatus - attr_accessor :status, :message - alias_method :allowed?, :status - - def initialize(status, message = '') - @status = status - @message = message - end - - def to_json(opts = nil) - { status: @status, message: @message }.to_json(opts) - end - end -end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 67eaa5e088d..1fe5155c093 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,5 +1,9 @@ module Gitlab class GitAccessWiki < GitAccess + ERROR_MESSAGES = { + write_to_wiki: "You are not allowed to write to this project's wiki." + }.freeze + def guest_can_download_code? Guest.can?(:download_wiki_code, project) end @@ -9,11 +13,11 @@ module Gitlab end def check_single_change_access(change) - if user_access.can_do_action?(:create_wiki) - build_status_object(true) - else - build_status_object(false, "You are not allowed to write to this project's wiki.") + unless user_access.can_do_action?(:create_wiki) + raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end + + true end end end diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index 0e14253ab4e..742118b76a8 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -13,6 +13,16 @@ module Gitlab super(identifier, project, revision) end + def changes_refs + return enum_for(:changes_refs) unless block_given? + + changes.each do |change| + oldrev, newrev, ref = change.strip.split(' ') + + yield oldrev, newrev, ref + end + end + private def deserialize_changes(changes) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 72466700c05..2343446bf22 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -2,6 +2,12 @@ require 'gitaly' module Gitlab module GitalyClient + module MigrationStatus + DISABLED = 1 + OPT_IN = 2 + OPT_OUT = 3 + end + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze MUTEX = Mutex.new @@ -46,8 +52,20 @@ module Gitlab Gitlab.config.gitaly.enabled end - def self.feature_enabled?(feature) - enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + def self.feature_enabled?(feature, status: MigrationStatus::OPT_IN) + return false if !enabled? || status == MigrationStatus::DISABLED + + feature = Feature.get("gitaly_#{feature}") + + # If the feature hasn't been set, turn it on if it's opt-out + return status == MigrationStatus::OPT_OUT unless Feature.persisted?(feature) + + if feature.percentage_of_time_value > 0 + # Probabilistically enable this feature + return Random.rand() * 100 < feature.percentage_of_time_value + end + + feature.enabled? end def self.migrate(feature) diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index 01cdc1ac14f..ba3da781dad 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,8 +5,6 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze - attr_accessor :stub - def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -23,24 +21,39 @@ module Gitlab stub.commit_is_ancestor(request).value end - class << self - def diff_from_parent(commit, options = {}) - repository = commit.project.repository - gitaly_repo = repository.gitaly_repository - stub = GitalyClient.stub(:diff, repository.storage) - parent = commit.parents[0] - parent_id = parent ? parent.id : EMPTY_TREE_ID - request = Gitaly::CommitDiffRequest.new( - repository: gitaly_repo, - left_commit_id: parent_id, - right_commit_id: commit.id, - ignore_whitespace_change: options.fetch(:ignore_whitespace_change, false), - paths: options.fetch(:paths, []), - ) - - Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + def diff_from_parent(commit, options = {}) + request_params = commit_diff_request_params(commit, options) + request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) + + response = diff_service_stub.commit_diff(Gitaly::CommitDiffRequest.new(request_params)) + Gitlab::Git::DiffCollection.new(GitalyClient::DiffStitcher.new(response), options) + end + + def commit_deltas(commit) + request_params = commit_diff_request_params(commit) + + response = diff_service_stub.commit_delta(Gitaly::CommitDeltaRequest.new(request_params)) + response.flat_map do |msg| + msg.deltas.map { |d| Gitlab::Git::Diff.new(d) } end end + + private + + def commit_diff_request_params(commit, options = {}) + parent_id = commit.parents[0]&.id || EMPTY_TREE_ID + + { + repository: @gitaly_repo, + left_commit_id: parent_id, + right_commit_id: commit.id, + paths: options.fetch(:paths, []) + } + end + + def diff_service_stub + GitalyClient.stub(:diff, @repository.storage) + end end end end diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb new file mode 100644 index 00000000000..1e117b7e74a --- /dev/null +++ b/lib/gitlab/gitaly_client/diff.rb @@ -0,0 +1,21 @@ +module Gitlab + module GitalyClient + class Diff + FIELDS = %i(from_path to_path old_mode new_mode from_id to_id patch).freeze + + attr_accessor(*FIELDS) + + def initialize(params) + params.each do |key, val| + public_send(:"#{key}=", val) + end + end + + def ==(other) + FIELDS.all? do |field| + public_send(field) == other.public_send(field) + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb new file mode 100644 index 00000000000..d84e8d752dc --- /dev/null +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -0,0 +1,31 @@ +module Gitlab + module GitalyClient + class DiffStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_diff = nil + + @rpc_response.each do |diff_msg| + if current_diff.nil? + diff_params = diff_msg.to_h.slice(*GitalyClient::Diff::FIELDS) + diff_params[:patch] = diff_msg.raw_patch_data + + current_diff = GitalyClient::Diff.new(diff_params) + else + current_diff.patch += diff_msg.raw_patch_data + end + + if diff_msg.end_of_patch + yield current_diff + current_diff = nil + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index 53c43e28df8..227fe45642e 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -44,6 +44,12 @@ module Gitlab branch_names.count end + def local_branches(sort_by: nil) + request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) + request.sort_by = sort_by_param(sort_by) if sort_by + consume_branches_response(stub.find_local_branches(request)) + end + private def consume_refs_response(response, prefix:) @@ -51,6 +57,16 @@ module Gitlab r.names.map { |name| name.sub(/\A#{Regexp.escape(prefix)}/, '') } end end + + def sort_by_param(sort_by) + enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) + raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + enum_value + end + + def consume_branches_response(response) + response.flat_map { |r| r.branches } + end end end end diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index 4acd297f5cb..86d055d3533 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -6,7 +6,7 @@ module Gitlab Gitaly::Repository.new( path: File.join(Gitlab.config.repositories.storages[repository_storage]['path'], relative_path), storage_name: repository_storage, - relative_path: relative_path, + relative_path: relative_path ) end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 1e09cb5ca11..319633656ff 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -1,3 +1,5 @@ +# rubocop:disable Metrics/AbcSize + module Gitlab module GonHelper def add_gon_variables @@ -13,11 +15,13 @@ module Gitlab gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION + gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') if current_user gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name + gon.current_user_avatar_url = current_user.avatar_url end end end diff --git a/lib/gitlab/google_code_import/client.rb b/lib/gitlab/google_code_import/client.rb index 890bd9a3554..b1dbf554e41 100644 --- a/lib/gitlab/google_code_import/client.rb +++ b/lib/gitlab/google_code_import/client.rb @@ -14,7 +14,7 @@ module Gitlab end def valid? - raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects") + raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.key?("projects") end def repos diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1b43440673c..ab38c0c3e34 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -95,7 +95,7 @@ module Gitlab labels = import_issue_labels(raw_issue) assignee_id = nil - if raw_issue.has_key?("owner") + if raw_issue.key?("owner") username = user_map[raw_issue["owner"]["name"]] if username.start_with?("@") @@ -144,7 +144,7 @@ module Gitlab def import_issue_comments(issue, comments) Note.transaction do while raw_comment = comments.shift - next if raw_comment.has_key?("deletedBy") + next if raw_comment.key?("deletedBy") content = format_content(raw_comment["content"]) updates = format_updates(raw_comment["updates"]) @@ -235,15 +235,15 @@ module Gitlab def format_updates(raw_updates) updates = [] - if raw_updates.has_key?("status") + if raw_updates.key?("status") updates << "*Status: #{raw_updates["status"]}*" end - if raw_updates.has_key?("owner") + if raw_updates.key?("owner") updates << "*Owner: #{user_map[raw_updates["owner"]]}*" end - if raw_updates.has_key?("cc") + if raw_updates.key?("cc") cc = raw_updates["cc"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -255,7 +255,7 @@ module Gitlab updates << "*Cc: #{cc.join(", ")}*" end - if raw_updates.has_key?("labels") + if raw_updates.key?("labels") labels = raw_updates["labels"].map do |l| deleted = l.start_with?("-") l = l[1..-1] if deleted @@ -267,11 +267,11 @@ module Gitlab updates << "*Labels: #{labels.join(", ")}*" end - if raw_updates.has_key?("mergedInto") + if raw_updates.key?("mergedInto") updates << "*Merged into: ##{raw_updates["mergedInto"]}*" end - if raw_updates.has_key?("blockedOn") + if raw_updates.key?("blockedOn") blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end @@ -279,7 +279,7 @@ module Gitlab updates << "*Blocked on: #{blocked_ons.join(", ")}*" end - if raw_updates.has_key?("blocking") + if raw_updates.key?("blocking") blockings = raw_updates["blocking"].map do |raw_blocked_on| format_blocking_updates(raw_blocked_on) end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index df962d203b7..e78b7f22e03 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -2,6 +2,9 @@ module Gitlab module HealthChecks class FsShardsCheck extend BaseAbstractCheck + RANDOM_STRING = SecureRandom.hex(1000).freeze + COMMAND_TIMEOUT = '1'.freeze + TIMEOUT_EXECUTABLE = 'timeout'.freeze class << self def readiness @@ -41,8 +44,6 @@ module Gitlab private - RANDOM_STRING = SecureRandom.hex(1000).freeze - def operation_metrics(ok_metric, latency_metric, operation, **labels) with_timing operation do |result, elapsed| [ @@ -63,8 +64,8 @@ module Gitlab @storage_paths ||= Gitlab.config.repositories.storages end - def with_timeout(args) - %w{timeout 1}.concat(args) + def exec_with_timeout(cmd_args, *args, &block) + Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) end def tmp_file_path(storage_name) @@ -78,7 +79,7 @@ module Gitlab def storage_stat_test(storage_name) stat_path = File.join(path(storage_name), '.') begin - _, status = Gitlab::Popen.popen(with_timeout(%W{ stat #{stat_path} })) + _, status = exec_with_timeout(%W{ stat #{stat_path} }) status == 0 rescue Errno::ENOENT File.exist?(stat_path) && File::Stat.new(stat_path).readable? @@ -86,7 +87,7 @@ module Gitlab end def storage_write_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ tee #{tmp_path} })) do |stdin| + _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -96,7 +97,7 @@ module Gitlab end def storage_read_test(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ diff #{tmp_path} - })) do |stdin| + _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| stdin.write(RANDOM_STRING) end status == 0 @@ -106,7 +107,7 @@ module Gitlab end def delete_test_file(tmp_path) - _, status = Gitlab::Popen.popen(with_timeout(%W{ rm -f #{tmp_path} })) + _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) status == 0 rescue Errno::ENOENT File.delete(tmp_path) rescue Errno::ENOENT diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb new file mode 100644 index 00000000000..b3c759b4730 --- /dev/null +++ b/lib/gitlab/health_checks/prometheus_text_format.rb @@ -0,0 +1,40 @@ +module Gitlab + module HealthChecks + class PrometheusTextFormat + def marshal(metrics) + "#{metrics_with_type_declarations(metrics).join("\n")}\n" + end + + private + + def metrics_with_type_declarations(metrics) + type_declaration_added = {} + + metrics.flat_map do |metric| + metric_lines = [] + + unless type_declaration_added.key?(metric.name) + type_declaration_added[metric.name] = true + metric_lines << metric_type_declaration(metric) + end + + metric_lines << metric_text(metric) + end + end + + def metric_type_declaration(metric) + "# TYPE #{metric.name} gauge" + end + + def metric_text(metric) + labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || '' + + if labels.empty? + "#{metric.name} #{metric.value}" + else + "#{metric.name}{#{labels}} #{metric.value}" + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d787d5db4a0..83bc230df3e 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -13,6 +13,8 @@ module Gitlab highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe) end + attr_reader :blob_name + def initialize(blob_name, blob_content, repository: nil) @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @@ -21,16 +23,9 @@ module Gitlab end def highlight(text, continue: true, plain: false) - if plain - hl_lexer = Rouge::Lexers::PlainText - continue = false - else - hl_lexer = self.lexer - end - - @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe - rescue - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + highlighted_text = highlight_text(text, continue: continue, plain: plain) + highlighted_text = link_dependencies(text, highlighted_text) if blob_name + highlighted_text end def lexer @@ -50,5 +45,27 @@ module Gitlab Rouge::Lexer.find_fancy(language_name) end + + def highlight_text(text, continue: true, plain: false) + if plain + highlight_plain(text) + else + highlight_rich(text, continue: continue) + end + end + + def highlight_plain(text) + @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + end + + def highlight_rich(text, continue: true) + @formatter.format(lexer.lex(text, continue: continue), tag: lexer.tag).html_safe + rescue + highlight_plain(text) + end + + def link_dependencies(text, highlighted_text) + Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3411516319f..f7ac48f7dbd 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -5,22 +5,46 @@ module Gitlab AVAILABLE_LANGUAGES = { 'en' => 'English', 'es' => 'Español', - 'de' => 'Deutsch' + 'de' => 'Deutsch', + 'zh_CN' => '简体中文', + 'zh_HK' => '繁體中文(香港)', + 'zh_TW' => '繁體中文(臺灣)' }.freeze def available_locales AVAILABLE_LANGUAGES.keys end - def set_locale(current_user) - requested_locale = current_user&.preferred_language || ::I18n.default_locale - locale = FastGettext.set_locale(requested_locale) - ::I18n.locale = locale + def locale + FastGettext.locale end - def reset_locale + def locale=(locale_string) + requested_locale = locale_string || ::I18n.default_locale + new_locale = FastGettext.set_locale(requested_locale) + ::I18n.locale = new_locale + end + + def use_default_locale FastGettext.set_locale(::I18n.default_locale) ::I18n.locale = ::I18n.default_locale end + + def with_locale(locale_string) + original_locale = locale + + self.locale = locale_string + yield + ensure + self.locale = original_locale + end + + def with_user_locale(user, &block) + with_locale(user&.preferred_language, &block) + end + + def with_default_locale(&block) + with_locale(::I18n.default_locale, &block) + end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d0f3cf2b514..ff2b1d08c3c 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -38,6 +38,7 @@ project_tree: - notes: - :author - :events + - :stages - :statuses - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e23a4715f..695852526cb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,6 +3,7 @@ module Gitlab class RelationFactory OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 3a7af363548..4a6091488c8 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -38,7 +38,7 @@ module Gitlab url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], headers: Hash.new { |h, k| h[k] = [] }, - created_at: created_at, + created_at: created_at } end end @@ -64,7 +64,7 @@ module Gitlab tty: true, stdin: true, stdout: true, - stderr: true, + stderr: true }.to_query + '&' + EXEC_COMMAND case url.scheme diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 46deea3cc9f..6fdf68641e2 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -39,7 +39,7 @@ module Gitlab def adapter_options opts = base_options.merge( - encryption: encryption, + encryption: encryption ) opts.merge!(auth_options) if has_auth? diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 2d5e47a6f3b..5e299e26c54 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -41,11 +41,6 @@ module Gitlab def update_user_attributes if persisted? - if auth_hash.has_email? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) @@ -55,10 +50,6 @@ module Gitlab # For an existing identity with no change in DN, this line changes nothing. identity.extern_uid = auth_hash.uid end - - gl_user.ldap_email = auth_hash.has_email? - - gl_user end def changed? @@ -69,6 +60,10 @@ module Gitlab ldap_config.block_auto_created_users end + def sync_email_from_provider? + true + end + def allowed? Gitlab::LDAP::Access.allowed?(gl_user) end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index c6dfa4ad9bd..4779755bb22 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,158 +1,10 @@ module Gitlab module Metrics - extend Gitlab::CurrentSettings - - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def self.settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + extend Gitlab::Metrics::InfluxDb + extend Gitlab::Metrics::Prometheus def self.enabled? - settings[:enabled] || false - end - - def self.mri? - RUBY_ENGINE == 'ruby' - end - - def self.method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - @method_call_threshold ||= settings[:method_call_threshold] - end - - def self.pool - @pool - end - - def self.submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError - end - end - end - end - - def self.prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def self.escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def self.measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) * 1000.0 - cpu_time = cpu_stop - cpu_start - - trans.increment("#{name}_real_time", real_time) - trans.increment("#{name}_cpu_time", cpu_time) - trans.increment("#{name}_call_count", 1) - - retval - end - - # Adds a tag to the current transaction (if any) - # - # name - The name of the tag to add. - # value - The value of the tag. - def self.tag_transaction(name, value) - trans = current_transaction - - trans&.add_tag(name, value) - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def self.action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def self.add_event(*args) - trans = current_transaction - - trans&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def self.series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - if enabled? - @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client. - new(udp: { host: host, port: port }) - end + influx_metrics_enabled? || prometheus_metrics_enabled? end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb new file mode 100644 index 00000000000..3a39791edbf --- /dev/null +++ b/lib/gitlab/metrics/influx_db.rb @@ -0,0 +1,170 @@ +module Gitlab + module Metrics + module InfluxDb + extend Gitlab::CurrentSettings + extend self + + MUTEX = Mutex.new + private_constant :MUTEX + + def influx_metrics_enabled? + settings[:enabled] || false + end + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 + } + end + + def mri? + RUBY_ENGINE == 'ruby' + end + + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= settings[:method_call_threshold] + end + + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end + end + end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) + end + + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def escape_value(value) + value.to_s.gsub('=', '\\=') + end + + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def tag_transaction(name, value) + trans = current_transaction + + trans&.add_tag(name, value) + end + + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction + + trans&.action = action + end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction + + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client. + new(udp: { host: host, port: port }) + end + end + end + @pool + end + end + end + end +end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb new file mode 100644 index 00000000000..3b5a2907195 --- /dev/null +++ b/lib/gitlab/metrics/null_metric.rb @@ -0,0 +1,10 @@ +module Gitlab + module Metrics + # Mocks ::Prometheus::Client::Metric and all derived metrics + class NullMetric + def method_missing(name, *args, &block) + nil + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb new file mode 100644 index 00000000000..60686509332 --- /dev/null +++ b/lib/gitlab/metrics/prometheus.rb @@ -0,0 +1,41 @@ +require 'prometheus/client' + +module Gitlab + module Metrics + module Prometheus + include Gitlab::CurrentSettings + + def prometheus_metrics_enabled? + @prometheus_metrics_enabled ||= current_application_settings[:prometheus_metrics_enabled] || false + end + + def registry + @registry ||= ::Prometheus::Client.registry + end + + def counter(name, docstring, base_labels = {}) + provide_metric(name) || registry.counter(name, docstring, base_labels) + end + + def summary(name, docstring, base_labels = {}) + provide_metric(name) || registry.summary(name, docstring, base_labels) + end + + def gauge(name, docstring, base_labels = {}) + provide_metric(name) || registry.gauge(name, docstring, base_labels) + end + + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets) + end + + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.new + end + end + end + end +end diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb index 9ad7a38d505..ac9d66c836d 100644 --- a/lib/gitlab/o_auth/provider.rb +++ b/lib/gitlab/o_auth/provider.rb @@ -22,7 +22,11 @@ module Gitlab def self.config_for(name) name = name.to_s if ldap_provider?(name) - Gitlab::LDAP::Config.new(name).options + if Gitlab::LDAP::Config.valid_provider?(name) + Gitlab::LDAP::Config.new(name).options + else + nil + end else Gitlab.config.omniauth.providers.find { |provider| provider.name == name } end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index afd24b4dcc5..7307f8c2c87 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,6 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash + update_email end def persisted? @@ -174,6 +175,22 @@ module Gitlab } end + def sync_email_from_provider? + auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + end + + def update_email + if auth_hash.has_email? && sync_email_from_provider? + if persisted? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end + + gl_user.external_email = true + gl_user.email_provider = auth_hash.provider + end + end + def log Gitlab::AppLogger end diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb new file mode 100644 index 00000000000..0d541935bc6 --- /dev/null +++ b/lib/gitlab/otp_key_rotator.rb @@ -0,0 +1,87 @@ +module Gitlab + # The +otp_key_base+ param is used to encrypt the User#otp_secret attribute. + # + # When +otp_key_base+ is changed, it invalidates the current encrypted values + # of User#otp_secret. This class can be used to decrypt all the values with + # the old key, encrypt them with the new key, and and update the database + # with the new values. + # + # For persistence between runs, a CSV file is used with the following columns: + # + # user_id, old_value, new_value + # + # Only the encrypted values are stored in this file. + # + # As users may have their 2FA settings changed at any time, this is only + # guaranteed to be safe if run offline. + class OtpKeyRotator + HEADERS = %w[user_id old_value new_value].freeze + + attr_reader :filename + + # Create a new rotator. +filename+ is used to store values by +calculate!+, + # and to update the database with new and old values in +apply!+ and + # +rollback!+, respectively. + def initialize(filename) + @filename = filename + end + + def rotate!(old_key:, new_key:) + old_key ||= Gitlab::Application.secrets.otp_key_base + + raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key + raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64 + + write_csv do |csv| + ActiveRecord::Base.transaction do + User.with_two_factor.in_batches do |relation| + rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) + rows.each do |row| + user = %i[id ciphertext iv salt].zip(row).to_h + new_value = reencrypt(user, old_key, new_key) + + User.where(id: user[:id]).update_all(encrypted_otp_secret: new_value) + csv << [user[:id], user[:ciphertext], new_value] + end + end + end + end + end + + def rollback! + ActiveRecord::Base.transaction do + CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| + User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value']) + end + end + end + + private + + attr_reader :old_key, :new_key + + def otp_secret_settings + @otp_secret_settings ||= User.encrypted_attributes[:otp_secret] + end + + def reencrypt(user, old_key, new_key) + original = user[:ciphertext].unpack("m").join + opts = { + iv: user[:iv].unpack("m").join, + salt: user[:salt].unpack("m").join, + algorithm: otp_secret_settings[:algorithm], + insecure_mode: otp_secret_settings[:insecure_mode] + } + + decrypted = Encryptor.decrypt(original, opts.merge(key: old_key)) + encrypted = Encryptor.encrypt(decrypted, opts.merge(key: new_key)) + [encrypted].pack("m") + end + + def write_csv(&blk) + File.open(filename, "w") do |file| + yield CSV.new(file, headers: HEADERS, write_headers: false) + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb new file mode 100644 index 00000000000..9ff6829cd49 --- /dev/null +++ b/lib/gitlab/path_regex.rb @@ -0,0 +1,265 @@ +module Gitlab + module PathRegex + extend self + + # All routes that appear on the top level must be listed here. + # This will make sure that groups cannot be created with these names + # as these routes would be masked by the paths already in place. + # + # Example: + # /api/api-project + # + # the path `api` shouldn't be allowed because it would be masked by `api/*` + # + TOP_LEVEL_ROUTES = %w[ + - + .well-known + abuse_reports + admin + all + api + assets + autocomplete + ci + dashboard + explore + files + groups + health_check + help + hooks + import + invites + issues + jwt + koding + member + merge_requests + new + notes + notification_settings + oauth + profile + projects + public + repository + robots.txt + s + search + sent_notifications + services + snippets + teams + u + unicorn_test + unsubscribes + uploads + users + ].freeze + + # This list should contain all words following `/*namespace_id/:project_id` in + # routes that contain a second wildcard. + # + # Example: + # /*namespace_id/:project_id/badges/*ref/build + # + # If `badges` was allowed as a project/group name, we would not be able to access the + # `badges` route for those projects: + # + # Consider a namespace with path `foo/bar` and a project called `badges`. + # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg` + # + # When accessing this path the route would be matched to the `badges` path + # with the following params: + # - namespace_id: `foo` + # - project_id: `bar` + # - ref: `badges/master` + # + # Failing to find the project, this would result in a 404. + # + # By rejecting `badges` the router can _count_ on the fact that `badges` will + # be preceded by the `namespace/project`. + PROJECT_WILDCARD_ROUTES = %w[ + - + badges + blame + blob + builds + commits + create + create_dir + edit + environments/folders + files + find_file + gitlab-lfs/objects + info/lfs/objects + new + preview + raw + refs + tree + update + wikis + ].freeze + + # These are all the paths that follow `/groups/*id/ or `/groups/*group_id` + # We need to reject these because we have a `/groups/*id` page that is the same + # as the `/*id`. + # + # If we would allow a subgroup to be created with the name `activity` then + # this group would not be accessible through `/groups/parent/activity` since + # this would map to the activity-page of its parent. + GROUP_ROUTES = %w[ + activity + analytics + audit_events + avatar + edit + group_members + hooks + issues + labels + ldap + ldap_group_links + merge_requests + milestones + notification_setting + pipeline_quota + projects + subgroups + ].freeze + + ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES + ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze + + # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validations, etc), `NAMESPACE_FORMAT_REGEX_JS` serves as a Javascript-compatible version of + # `NAMESPACE_FORMAT_REGEX`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + + NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze + NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze + PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze + FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/)*#{NAMESPACE_FORMAT_REGEX}}.freeze + + def root_namespace_route_regex + @root_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{NAMESPACE_FORMAT_REGEX} + }x + end + end + + def full_namespace_route_regex + @full_namespace_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + #{root_namespace_route_regex} + (?: + / + (?!#{illegal_words}/) + #{NAMESPACE_FORMAT_REGEX} + )* + }x + end + end + + def project_route_regex + @project_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{PROJECT_PATH_FORMAT_REGEX} + }x + end + end + + def project_git_route_regex + @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze + end + + def root_namespace_path_regex + @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} + end + + def full_namespace_path_regex + @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} + end + + def project_path_regex + @project_path_regex ||= %r{\A#{project_route_regex}/\z} + end + + def full_project_path_regex + @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} + end + + def full_namespace_format_regex + @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_regex + @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/.freeze + end + + def namespace_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ + end + + def project_path_format_regex + @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/.freeze + end + + def project_path_format_message + "can contain only letters, digits, '_', '-' and '.'. " \ + "Cannot start with '-', end in '.git' or end in '.atom'" \ + end + + def archive_formats_regex + # |zip|tar| tar.gz | tar.bz2 | + @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze + end + + def git_reference_regex + # Valid git ref regex, see: + # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html + + @git_reference_regex ||= single_line_regexp %r{ + (?! + (?# doesn't begins with) + \/| (?# rule #6) + (?# doesn't contain) + .*(?: + [\/.]\.| (?# rule #1,3) + \/\/| (?# rule #6) + @\{| (?# rule #8) + \\ (?# rule #9) + ) + ) + [^\000-\040\177~^:?*\[]+ (?# rule #4-5) + (?# doesn't end with) + (?<!\.lock) (?# rule #1) + (?<![\/.]) (?# rule #6-7) + }x + end + + private + + def single_line_regexp(regex) + # Turns a multiline extended regexp into a single line one, + # beacuse `rake routes` breaks on multiline regexes. + Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 47cfe412715..561aa9e162c 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -84,23 +84,7 @@ module Gitlab def blobs return [] unless Ability.allowed?(@current_user, :download_code, @project) - @blobs ||= begin - blobs = project.repository.search_files_by_content(query, repository_ref).first(100) - found_file_names = Set.new - - results = blobs.map do |blob| - blob = self.class.parse_search_result(blob) - found_file_names << blob.filename - - [blob.filename, blob] - end - - project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| - results << [filename, nil] unless found_file_names.include?(filename) - end - - results.sort_by(&:first) - end + @blobs ||= Gitlab::FileFinder.new(project, repository_ref).find(query) end def wiki_blobs diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb new file mode 100644 index 00000000000..2a2eb4ae57f --- /dev/null +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -0,0 +1,26 @@ +module Gitlab + module Prometheus + module Queries + class BaseQuery + attr_accessor :client + delegate :query_range, :query, to: :client, prefix: true + + def raw_memory_usage_query(environment_slug) + %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} + end + + def raw_cpu_usage_query(environment_slug) + %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} + end + + def initialize(client) + @client = client + end + + def query(*args) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb new file mode 100644 index 00000000000..2cc08731f8d --- /dev/null +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -0,0 +1,26 @@ +module Gitlab::Prometheus::Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug + + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), + + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb new file mode 100644 index 00000000000..01d756d7284 --- /dev/null +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -0,0 +1,20 @@ +module Gitlab::Prometheus::Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f + + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) + + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus_client.rb index 37125980b1c..5b51a1779dd 100644 --- a/lib/gitlab/prometheus.rb +++ b/lib/gitlab/prometheus_client.rb @@ -2,7 +2,7 @@ module Gitlab PrometheusError = Class.new(StandardError) # Helper methods to interact with Prometheus network services & resources - class Prometheus + class PrometheusClient attr_reader :api_url def initialize(api_url:) @@ -15,7 +15,7 @@ module Gitlab def query(query, time: Time.now) get_result('vector') do - json_api_get('query', query: query, time: time.utc.to_f) + json_api_get('query', query: query, time: time.to_f) end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 34b6921d606..009ecc9b263 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,204 +2,6 @@ module Gitlab module Regex extend self - # All routes that appear on the top level must be listed here. - # This will make sure that groups cannot be created with these names - # as these routes would be masked by the paths already in place. - # - # Example: - # /api/api-project - # - # the path `api` shouldn't be allowed because it would be masked by `api/*` - # - TOP_LEVEL_ROUTES = %w[ - - - .well-known - abuse_reports - admin - all - api - assets - autocomplete - ci - dashboard - explore - files - groups - health_check - help - hooks - import - invites - issues - jwt - koding - member - merge_requests - new - notes - notification_settings - oauth - profile - projects - public - repository - robots.txt - s - search - sent_notifications - services - snippets - system - teams - u - unicorn_test - unsubscribes - uploads - users - ].freeze - - # This list should contain all words following `/*namespace_id/:project_id` in - # routes that contain a second wildcard. - # - # Example: - # /*namespace_id/:project_id/badges/*ref/build - # - # If `badges` was allowed as a project/group name, we would not be able to access the - # `badges` route for those projects: - # - # Consider a namespace with path `foo/bar` and a project called `badges`. - # The route to the build badge would then be `/foo/bar/badges/badges/master/build.svg` - # - # When accessing this path the route would be matched to the `badges` path - # with the following params: - # - namespace_id: `foo` - # - project_id: `bar` - # - ref: `badges/master` - # - # Failing to find the project, this would result in a 404. - # - # By rejecting `badges` the router can _count_ on the fact that `badges` will - # be preceded by the `namespace/project`. - PROJECT_WILDCARD_ROUTES = %w[ - badges - blame - blob - builds - commits - create - create_dir - edit - environments/folders - files - find_file - gitlab-lfs/objects - info/lfs/objects - new - preview - raw - refs - tree - update - wikis - ].freeze - - # These are all the paths that follow `/groups/*id/ or `/groups/*group_id` - # We need to reject these because we have a `/groups/*id` page that is the same - # as the `/*id`. - # - # If we would allow a subgroup to be created with the name `activity` then - # this group would not be accessible through `/groups/parent/activity` since - # this would map to the activity-page of its parent. - GROUP_ROUTES = %w[ - activity - analytics - audit_events - avatar - edit - group_members - hooks - issues - labels - ldap - ldap_group_links - merge_requests - milestones - notification_setting - pipeline_quota - projects - subgroups - ].freeze - - ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES - ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze - - # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript - # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. - # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to - # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of - # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation - # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze - NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze - NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze - NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze - PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze - - # Same as NAMESPACE_REGEX_STR but allows `/` in the path. - # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR - FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze - - def root_namespace_route_regex - @root_namespace_route_regex ||= begin - illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE) - - single_line_regexp %r{ - (?!(#{illegal_words})/) - #{NAMESPACE_REGEX_STR} - }x - end - end - - def root_namespace_path_regex - @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} - end - - def full_namespace_path_regex - @full_namespace_path_regex ||= %r{\A#{namespace_route_regex}/\z} - end - - def full_project_path_regex - @full_project_path_regex ||= %r{\A#{namespace_route_regex}/#{project_route_regex}/\z} - end - - def namespace_regex - @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze - end - - def full_namespace_regex - @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} - end - - def namespace_route_regex - @namespace_route_regex ||= begin - illegal_words = Regexp.new(Regexp.union(ILLEGAL_GROUP_PATH_WORDS).source, Regexp::IGNORECASE) - - single_line_regexp %r{ - #{root_namespace_route_regex} - (?: - / - (?!#{illegal_words}/) - #{NAMESPACE_REGEX_STR} - )* - }x - end - end - - def namespace_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.', '.git' or '.atom'." \ - end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -217,34 +19,6 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end - def project_path_regex - @project_path_regex ||= %r{\A#{project_route_regex}/\z} - end - - def project_route_regex - @project_route_regex ||= begin - illegal_words = Regexp.new(Regexp.union(ILLEGAL_PROJECT_PATH_WORDS).source, Regexp::IGNORECASE) - - single_line_regexp %r{ - (?!(#{illegal_words})/) - #{PROJECT_REGEX_STR} - }x - end - end - - def project_git_route_regex - @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze - end - - def project_path_format_regex - @project_path_format_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze - end - - def project_path_regex_message - "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-', end in '.git' or end in '.atom'" \ - end - def file_name_regex @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end @@ -253,36 +27,8 @@ module Gitlab "can contain only letters, digits, '_', '-', '@', '+' and '.'." end - def archive_formats_regex - # |zip|tar| tar.gz | tar.bz2 | - @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze - end - - def git_reference_regex - # Valid git ref regex, see: - # https://www.kernel.org/pub/software/scm/git/docs/git-check-ref-format.html - - @git_reference_regex ||= single_line_regexp %r{ - (?! - (?# doesn't begins with) - \/| (?# rule #6) - (?# doesn't contain) - .*(?: - [\/.]\.| (?# rule #1,3) - \/\/| (?# rule #6) - @\{| (?# rule #8) - \\ (?# rule #9) - ) - ) - [^\000-\040\177~^:?*\[]+ (?# rule #4-5) - (?# doesn't end with) - (?<!\.lock) (?# rule #1) - (?<![\/.]) (?# rule #6-7) - }x - end - def container_registry_reference_regex - git_reference_regex + Gitlab::PathRegex.git_reference_regex end ## diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index 36791fae60f..877aa6e6a28 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -25,8 +25,8 @@ module Gitlab def parse_entry(entry) raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) - raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') - raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + raise FormatError, 'Route map entry does not have a source key' unless entry.key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.key?('public') source_pattern = entry['source'] public_path = entry['public'] diff --git a/lib/gitlab/routes/legacy_builds.rb b/lib/gitlab/routes/legacy_builds.rb new file mode 100644 index 00000000000..36d1a8a6f64 --- /dev/null +++ b/lib/gitlab/routes/legacy_builds.rb @@ -0,0 +1,36 @@ +module Gitlab + module Routes + class LegacyBuilds + def initialize(map) + @map = map + end + + def draw + @map.instance_eval do + resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + resources :artifacts, only: [], controller: 'build_artifacts' do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end + end + end + + member do + get :raw + end + + resource :artifacts, only: [], controller: 'build_artifacts' do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + end + end + end + end + end + end +end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 117fc508135..2442c2ded3b 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -11,7 +11,7 @@ module Gitlab Raven.user_context( id: current_user.id, email: current_user.email, - username: current_user.username, + username: current_user.username ) end end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb index 12a385f90fd..caab8856014 100644 --- a/lib/gitlab/slash_commands/command_definition.rb +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -48,17 +48,23 @@ module Gitlab end def to_h(opts) + context = OpenStruct.new(opts) + desc = description if desc.respond_to?(:call) - context = OpenStruct.new(opts) desc = context.instance_exec(&desc) rescue '' end + prms = params + if prms.respond_to?(:call) + prms = Array(context.instance_exec(&prms)) rescue params + end + { name: name, aliases: aliases, description: desc, - params: params + params: prms } end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb index 614bafbe1b2..1b5b4566d81 100644 --- a/lib/gitlab/slash_commands/dsl.rb +++ b/lib/gitlab/slash_commands/dsl.rb @@ -40,8 +40,8 @@ module Gitlab # command :command_key do |arguments| # # Awesome code block # end - def params(*params) - @params = params + def params(*params, &block) + @params = block_given? ? block : params end # Allows to give an explanation of what the command will do when diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb new file mode 100644 index 00000000000..94fba0a221a --- /dev/null +++ b/lib/gitlab/string_range_marker.rb @@ -0,0 +1,102 @@ +module Gitlab + class StringRangeMarker + attr_accessor :raw_line, :rich_line + + def initialize(raw_line, rich_line = raw_line) + @raw_line = raw_line + @rich_line = ERB::Util.html_escape(rich_line) + end + + def mark(marker_ranges) + return rich_line unless marker_ranges + + rich_marker_ranges = [] + marker_ranges.each do |range| + # Map the inline-diff range based on the raw line to character positions in the rich line + rich_positions = position_mapping[range].flatten + # Turn the array of character positions into ranges + rich_marker_ranges.concat(collapse_ranges(rich_positions)) + end + + offset = 0 + # Mark each range + rich_marker_ranges.each_with_index do |range, i| + offset_range = (range.begin + offset)..(range.end + offset) + original_text = rich_line[offset_range] + + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + + rich_line[offset_range] = text + + offset += text.length - original_text.length + end + + rich_line.html_safe + end + + private + + # Mapping of character positions in the raw line, to the rich (highlighted) line + def position_mapping + @position_mapping ||= begin + mapping = [] + rich_pos = 0 + (0..raw_line.length).each do |raw_pos| + rich_char = rich_line[rich_pos] + + # The raw and rich lines are the same except for HTML tags, + # so skip over any `<...>` segment + while rich_char == '<' + until rich_char == '>' + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + rich_pos += 1 + rich_char = rich_line[rich_pos] + end + + # multi-char HTML entities in the rich line correspond to a single character in the raw line + if rich_char == '&' + multichar_mapping = [rich_pos] + until rich_char == ';' + rich_pos += 1 + multichar_mapping << rich_pos + rich_char = rich_line[rich_pos] + end + + mapping[raw_pos] = multichar_mapping + else + mapping[raw_pos] = rich_pos + end + + rich_pos += 1 + end + + mapping + end + end + + # Takes an array of integers, and returns an array of ranges covering the same integers + def collapse_ranges(positions) + return [] if positions.empty? + ranges = [] + + start = prev = positions[0] + range = start..prev + positions[1..-1].each do |pos| + if pos == prev + 1 + range = start..pos + prev = pos + else + ranges << range + start = prev = pos + range = start..prev + end + end + ranges << range + + ranges + end + end +end diff --git a/lib/gitlab/string_regex_marker.rb b/lib/gitlab/string_regex_marker.rb new file mode 100644 index 00000000000..7ebf1c0428c --- /dev/null +++ b/lib/gitlab/string_regex_marker.rb @@ -0,0 +1,13 @@ +module Gitlab + class StringRegexMarker < StringRangeMarker + def mark(regex, group: 0, &block) + regex_match = raw_line.match(regex) + return rich_line unless regex_match + + begin_index, end_index = regex_match.offset(group) + name_range = begin_index..(end_index - 1) + + super([name_range], &block) + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index ccb456bcc94..23af9318d1a 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -61,7 +61,12 @@ module Gitlab elsif object.for_snippet? snippet = Snippet.find(object.noteable_id) - project_snippet_url(snippet, anchor: dom_id(object)) + + if snippet.is_a?(PersonalSnippet) + snippet_url(snippet, anchor: dom_id(object)) + else + project_snippet_url(snippet, anchor: dom_id(object)) + end end end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 9ce13feb79a..c81dc7e30d0 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -18,12 +18,6 @@ module Gitlab false end - def self.http_credentials_for_user(user) - return {} unless user.respond_to?(:username) - - { user: user.username } - end - def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index e46ff313654..3b922da7ced 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -38,6 +38,16 @@ module Gitlab end end + def can_delete_branch?(ref) + return false unless can_access_git? + + if ProtectedBranch.protected?(project, ref) + user.can?(:delete_protected_branch, project) + else + user.can?(:push_code, project) + end + end + def can_push_to_branch?(ref) return false unless can_access_git? diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 4c395b4266e..fa182c4deda 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -21,5 +21,13 @@ module Gitlab nil end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end end end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 2e31f4462f9..2b53798e70f 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -41,9 +41,9 @@ module Gitlab def options { - 'Private' => PRIVATE, - 'Internal' => INTERNAL, - 'Public' => PUBLIC + N_('VisibilityLevel|Private') => PRIVATE, + N_('VisibilityLevel|Internal') => INTERNAL, + N_('VisibilityLevel|Public') => PUBLIC } end @@ -83,7 +83,7 @@ module Gitlab end def valid_level?(level) - options.has_value?(level) + options.value?(level) end def level_name(level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 72875bdaa17..7f27317775c 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,7 +22,7 @@ module Gitlab params = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), - RepoPath: repo_path, + RepoPath: repo_path } if Gitlab.config.gitaly.enabled @@ -31,8 +31,7 @@ module Gitlab feature_enabled = case action.to_s when 'git_receive_pack' - # Disabled for now, see https://gitlab.com/gitlab-org/gitaly/issues/172 - false + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) when 'git_upload_pack' Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) when 'info_refs' @@ -51,7 +50,7 @@ module Gitlab { StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", LfsOid: oid, - LfsSize: size, + LfsSize: size } end @@ -62,7 +61,7 @@ module Gitlab def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, - 'BlobId' => blob.id, + 'BlobId' => blob.id } [ @@ -127,10 +126,10 @@ module Gitlab 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], 'Header' => terminal[:headers], - 'MaxSessionTime' => terminal[:max_session_time], + 'MaxSessionTime' => terminal[:max_session_time] } } - details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.key?(:ca_pem) details end @@ -165,7 +164,7 @@ module Gitlab encoded_message, secret, true, - { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' } ) end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb index 80784adfd76..939b23a3421 100644 --- a/lib/rouge/lexers/math.rb +++ b/lib/rouge/lexers/math.rb @@ -1,21 +1,9 @@ module Rouge module Lexers - class Math < Lexer + class Math < PlainText title "A passthrough lexer used for LaTeX input" - desc "A boring lexer that doesn't highlight anything" - + desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" tag 'math' - mimetypes 'text/plain' - - default_options token: 'Text' - - def token - @token ||= Token[option :token] - end - - def stream_tokens(string, &b) - yield self.token, string - end end end end diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb index 7d5700b7f6d..63c461764fc 100644 --- a/lib/rouge/lexers/plantuml.rb +++ b/lib/rouge/lexers/plantuml.rb @@ -1,21 +1,9 @@ module Rouge module Lexers - class Plantuml < Lexer + class Plantuml < PlainText title "A passthrough lexer used for PlantUML input" - desc "A boring lexer that doesn't highlight anything" - + desc "PLEASE REFACTOR - this should be handled by SyntaxHighlightFilter" tag 'plantuml' - mimetypes 'text/plain' - - default_options token: 'Text' - - def token - @token ||= Token[option :token] - end - - def stream_tokens(string, &b) - yield self.token, string - end end end end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 6e351365de0..c5f93336346 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -48,7 +48,7 @@ gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" -gitaly_enabled=false +gitaly_enabled=true gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 9472c3c992f..295c79fccfc 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -86,5 +86,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # This variable controls whether the init script starts/stops Gitaly -gitaly_enabled=false +gitaly_enabled=true +gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) +gitaly_pid_path="$pid_path/gitaly.pid" gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/system_check.rb b/lib/system_check.rb new file mode 100644 index 00000000000..466c39904fa --- /dev/null +++ b/lib/system_check.rb @@ -0,0 +1,21 @@ +# Library to perform System Checks +# +# Every Check is implemented as its own class inherited from SystemCheck::BaseCheck +# Execution coordination and boilerplate output is done by the SystemCheck::SimpleExecutor +# +# This structure decouples checks from Rake tasks and facilitates unit-testing +module SystemCheck + # Executes a bunch of checks for specified component + # + # @param [String] component name of the component relative to the checks being executed + # @param [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + def self.run(component, checks = []) + executor = SimpleExecutor.new(component) + + checks.each do |check| + executor << check + end + + executor.execute + end +end diff --git a/lib/system_check/app/active_users_check.rb b/lib/system_check/app/active_users_check.rb new file mode 100644 index 00000000000..1d72c8d6903 --- /dev/null +++ b/lib/system_check/app/active_users_check.rb @@ -0,0 +1,17 @@ +module SystemCheck + module App + class ActiveUsersCheck < SystemCheck::BaseCheck + set_name 'Active users:' + + def multi_check + active_users = User.active.count + + if active_users > 0 + $stdout.puts active_users.to_s.color(:green) + else + $stdout.puts active_users.to_s.color(:red) + end + end + end + end +end diff --git a/lib/system_check/app/database_config_exists_check.rb b/lib/system_check/app/database_config_exists_check.rb new file mode 100644 index 00000000000..d1fae192350 --- /dev/null +++ b/lib/system_check/app/database_config_exists_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class DatabaseConfigExistsCheck < SystemCheck::BaseCheck + set_name 'Database config exists?' + + def check? + database_config_file = Rails.root.join('config', 'database.yml') + + File.exist?(database_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/database.yml.<your db> to config/database.yml', + 'Check that the information in config/database.yml is correct' + ) + for_more_information( + 'doc/install/databases.md', + 'http://guides.rubyonrails.org/getting_started.html#configuring-a-database' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/git_config_check.rb b/lib/system_check/app/git_config_check.rb new file mode 100644 index 00000000000..198867f7ac6 --- /dev/null +++ b/lib/system_check/app/git_config_check.rb @@ -0,0 +1,42 @@ +module SystemCheck + module App + class GitConfigCheck < SystemCheck::BaseCheck + OPTIONS = { + 'core.autocrlf' => 'input' + }.freeze + + set_name 'Git configured correctly?' + + def check? + correct_options = OPTIONS.map do |name, value| + run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value + end + + correct_options.all? + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successful (according to their exit code) + # Returns false if any or all subcommands failed. + def repair! + return false unless is_gitlab_user? + + command_success = OPTIONS.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + end + + def show_error + try_fixing_it( + sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{OPTIONS['core.autocrlf']}\"") + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + end + end + end +end diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb new file mode 100644 index 00000000000..c388682dfb4 --- /dev/null +++ b/lib/system_check/app/git_version_check.rb @@ -0,0 +1,29 @@ +module SystemCheck + module App + class GitVersionCheck < SystemCheck::BaseCheck + set_name -> { "Git version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 7, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + $stdout.puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" + + try_fixing_it( + "Update your git to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_exists_check.rb b/lib/system_check/app/gitlab_config_exists_check.rb new file mode 100644 index 00000000000..247aa0994e4 --- /dev/null +++ b/lib/system_check/app/gitlab_config_exists_check.rb @@ -0,0 +1,24 @@ +module SystemCheck + module App + class GitlabConfigExistsCheck < SystemCheck::BaseCheck + set_name 'GitLab config exists?' + + def check? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + + File.exist?(gitlab_config_file) + end + + def show_error + try_fixing_it( + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/gitlab_config_up_to_date_check.rb b/lib/system_check/app/gitlab_config_up_to_date_check.rb new file mode 100644 index 00000000000..c609e48e133 --- /dev/null +++ b/lib/system_check/app/gitlab_config_up_to_date_check.rb @@ -0,0 +1,30 @@ +module SystemCheck + module App + class GitlabConfigUpToDateCheck < SystemCheck::BaseCheck + set_name 'GitLab config up to date?' + set_skip_reason "can't check because of previous errors" + + def skip? + gitlab_config_file = Rails.root.join('config', 'gitlab.yml') + !File.exist?(gitlab_config_file) + end + + def check? + # omniauth or ldap could have been deleted from the file + !Gitlab.config['git_host'] + end + + def show_error + try_fixing_it( + 'Back-up your config/gitlab.yml', + 'Copy config/gitlab.yml.example to config/gitlab.yml', + 'Update config/gitlab.yml to match your setup' + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_exists_check.rb b/lib/system_check/app/init_script_exists_check.rb new file mode 100644 index 00000000000..d246e058e86 --- /dev/null +++ b/lib/system_check/app/init_script_exists_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class InitScriptExistsCheck < SystemCheck::BaseCheck + set_name 'Init script exists?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def check? + script_path = '/etc/init.d/gitlab' + File.exist?(script_path) + end + + def show_error + try_fixing_it( + 'Install the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/init_script_up_to_date_check.rb b/lib/system_check/app/init_script_up_to_date_check.rb new file mode 100644 index 00000000000..015c7ed1731 --- /dev/null +++ b/lib/system_check/app/init_script_up_to_date_check.rb @@ -0,0 +1,43 @@ +module SystemCheck + module App + class InitScriptUpToDateCheck < SystemCheck::BaseCheck + SCRIPT_PATH = '/etc/init.d/gitlab'.freeze + + set_name 'Init script up-to-date?' + set_skip_reason 'skipped (omnibus-gitlab has no init script)' + + def skip? + omnibus_gitlab? + end + + def multi_check + recipe_path = Rails.root.join('lib/support/init.d/', 'gitlab') + + unless File.exist?(SCRIPT_PATH) + $stdout.puts "can't check because of previous errors".color(:magenta) + return + end + + recipe_content = File.read(recipe_path) + script_content = File.read(SCRIPT_PATH) + + if recipe_content == script_content + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + + def show_error + try_fixing_it( + 'Re-download the init script' + ) + for_more_information( + see_installation_guide_section 'Install Init Script' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/log_writable_check.rb b/lib/system_check/app/log_writable_check.rb new file mode 100644 index 00000000000..3e0c436d6ee --- /dev/null +++ b/lib/system_check/app/log_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class LogWritableCheck < SystemCheck::BaseCheck + set_name 'Log directory writable?' + + def check? + File.writable?(log_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{log_path}", + "sudo chmod -R u+rwX #{log_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def log_path + Rails.root.join('log') + end + end + end +end diff --git a/lib/system_check/app/migrations_are_up_check.rb b/lib/system_check/app/migrations_are_up_check.rb new file mode 100644 index 00000000000..5eedbacce77 --- /dev/null +++ b/lib/system_check/app/migrations_are_up_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class MigrationsAreUpCheck < SystemCheck::BaseCheck + set_name 'All migrations up?' + + def check? + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) + + migration_status !~ /down\s+\d{14}/ + end + + def show_error + try_fixing_it( + sudo_gitlab('bundle exec rake db:migrate RAILS_ENV=production') + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/orphaned_group_members_check.rb b/lib/system_check/app/orphaned_group_members_check.rb new file mode 100644 index 00000000000..2b46d36fe51 --- /dev/null +++ b/lib/system_check/app/orphaned_group_members_check.rb @@ -0,0 +1,20 @@ +module SystemCheck + module App + class OrphanedGroupMembersCheck < SystemCheck::BaseCheck + set_name 'Database contains orphaned GroupMembers?' + set_check_pass 'no' + set_check_fail 'yes' + + def check? + !GroupMember.where('user_id not in (select id from users)').exists? + end + + def show_error + try_fixing_it( + 'You can delete the orphaned records using something along the lines of:', + sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") + ) + end + end + end +end diff --git a/lib/system_check/app/projects_have_namespace_check.rb b/lib/system_check/app/projects_have_namespace_check.rb new file mode 100644 index 00000000000..a6ec9f7665c --- /dev/null +++ b/lib/system_check/app/projects_have_namespace_check.rb @@ -0,0 +1,37 @@ +module SystemCheck + module App + class ProjectsHaveNamespaceCheck < SystemCheck::BaseCheck + set_name 'Projects have namespace:' + set_skip_reason "can't check, you have no projects" + + def skip? + !Project.exists? + end + + def multi_check + $stdout.puts '' + + Project.find_each(batch_size: 100) do |project| + $stdout.print sanitized_message(project) + + if project.namespace + $stdout.puts 'yes'.color(:green) + else + $stdout.puts 'no'.color(:red) + show_error + end + end + end + + def show_error + try_fixing_it( + "Migrate global projects" + ) + for_more_information( + "doc/update/5.4-to-6.0.md in section \"#global-projects\"" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb new file mode 100644 index 00000000000..a0610e73576 --- /dev/null +++ b/lib/system_check/app/redis_version_check.rb @@ -0,0 +1,25 @@ +module SystemCheck + module App + class RedisVersionCheck < SystemCheck::BaseCheck + MIN_REDIS_VERSION = '2.8.0'.freeze + set_name "Redis version >= #{MIN_REDIS_VERSION}?" + + def check? + redis_version = run_command(%w(redis-cli --version)) + redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) + + redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(MIN_REDIS_VERSION)) + end + + def show_error + try_fixing_it( + "Update your redis server to a version >= #{MIN_REDIS_VERSION}" + ) + for_more_information( + 'gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb new file mode 100644 index 00000000000..fd82f5f8a4a --- /dev/null +++ b/lib/system_check/app/ruby_version_check.rb @@ -0,0 +1,27 @@ +module SystemCheck + module App + class RubyVersionCheck < SystemCheck::BaseCheck + set_name -> { "Ruby version >= #{self.required_version} ?" } + set_check_pass -> { "yes (#{self.current_version})" } + + def self.required_version + @required_version ||= Gitlab::VersionInfo.new(2, 3, 3) + end + + def self.current_version + @current_version ||= Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) + end + + def check? + self.class.current_version.valid? && self.class.required_version <= self.class.current_version + end + + def show_error + try_fixing_it( + "Update your ruby to a version >= #{self.class.required_version} from #{self.class.current_version}" + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/tmp_writable_check.rb b/lib/system_check/app/tmp_writable_check.rb new file mode 100644 index 00000000000..99a75e57abf --- /dev/null +++ b/lib/system_check/app/tmp_writable_check.rb @@ -0,0 +1,28 @@ +module SystemCheck + module App + class TmpWritableCheck < SystemCheck::BaseCheck + set_name 'Tmp directory writable?' + + def check? + File.writable?(tmp_path) + end + + def show_error + try_fixing_it( + "sudo chown -R gitlab #{tmp_path}", + "sudo chmod -R u+rwX #{tmp_path}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def tmp_path + Rails.root.join('tmp') + end + end + end +end diff --git a/lib/system_check/app/uploads_directory_exists_check.rb b/lib/system_check/app/uploads_directory_exists_check.rb new file mode 100644 index 00000000000..7026d0ba075 --- /dev/null +++ b/lib/system_check/app/uploads_directory_exists_check.rb @@ -0,0 +1,21 @@ +module SystemCheck + module App + class UploadsDirectoryExistsCheck < SystemCheck::BaseCheck + set_name 'Uploads directory exists?' + + def check? + File.directory?(Rails.root.join('public/uploads')) + end + + def show_error + try_fixing_it( + "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + end + end +end diff --git a/lib/system_check/app/uploads_path_permission_check.rb b/lib/system_check/app/uploads_path_permission_check.rb new file mode 100644 index 00000000000..7df6c060254 --- /dev/null +++ b/lib/system_check/app/uploads_path_permission_check.rb @@ -0,0 +1,36 @@ +module SystemCheck + module App + class UploadsPathPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory has correct permissions?' + set_skip_reason 'skipped (no uploads folder found)' + + def skip? + !File.directory?(rails_uploads_path) + end + + def check? + File.stat(uploads_fullpath).mode == 040700 + end + + def show_error + try_fixing_it( + "sudo chmod 700 #{uploads_fullpath}" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def rails_uploads_path + Rails.root.join('public/uploads') + end + + def uploads_fullpath + File.realpath(rails_uploads_path) + end + end + end +end diff --git a/lib/system_check/app/uploads_path_tmp_permission_check.rb b/lib/system_check/app/uploads_path_tmp_permission_check.rb new file mode 100644 index 00000000000..b276a81eac1 --- /dev/null +++ b/lib/system_check/app/uploads_path_tmp_permission_check.rb @@ -0,0 +1,40 @@ +module SystemCheck + module App + class UploadsPathTmpPermissionCheck < SystemCheck::BaseCheck + set_name 'Uploads directory tmp has correct permissions?' + set_skip_reason 'skipped (no tmp uploads folder yet)' + + def skip? + !File.directory?(uploads_fullpath) || !Dir.exist?(upload_path_tmp) + end + + def check? + # If tmp upload dir has incorrect permissions, assume others do as well + # Verify drwx------ permissions + File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) + end + + def show_error + try_fixing_it( + "sudo chown -R #{gitlab_user} #{uploads_fullpath}", + "sudo find #{uploads_fullpath} -type f -exec chmod 0644 {} \\;", + "sudo find #{uploads_fullpath} -type d -not -path #{uploads_fullpath} -exec chmod 0700 {} \\;" + ) + for_more_information( + see_installation_guide_section 'GitLab' + ) + fix_and_rerun + end + + private + + def upload_path_tmp + File.join(uploads_fullpath, 'tmp') + end + + def uploads_fullpath + File.realpath(Rails.root.join('public/uploads')) + end + end + end +end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb new file mode 100644 index 00000000000..5dcb3f0886b --- /dev/null +++ b/lib/system_check/base_check.rb @@ -0,0 +1,129 @@ +module SystemCheck + # Base class for Checks. You must inherit from here + # and implement the methods below when necessary + class BaseCheck + include ::SystemCheck::Helpers + + # Define a custom term for when check passed + # + # @param [String] term used when check passed (default: 'yes') + def self.set_check_pass(term) + @check_pass = term + end + + # Define a custom term for when check failed + # + # @param [String] term used when check failed (default: 'no') + def self.set_check_fail(term) + @check_fail = term + end + + # Define the name of the SystemCheck that will be displayed during execution + # + # @param [String] name of the check + def self.set_name(name) + @name = name + end + + # Define the reason why we skipped the SystemCheck + # + # This is only used if subclass implements `#skip?` + # + # @param [String] reason to be displayed + def self.set_skip_reason(reason) + @skip_reason = reason + end + + # Term to be displayed when check passed + # + # @return [String] term when check passed ('yes' if not re-defined in a subclass) + def self.check_pass + call_or_return(@check_pass) || 'yes' + end + + ## Term to be displayed when check failed + # + # @return [String] term when check failed ('no' if not re-defined in a subclass) + def self.check_fail + call_or_return(@check_fail) || 'no' + end + + # Name of the SystemCheck defined by the subclass + # + # @return [String] the name + def self.display_name + call_or_return(@name) || self.name + end + + # Skip reason defined by the subclass + # + # @return [String] the reason + def self.skip_reason + call_or_return(@skip_reason) || 'skipped' + end + + # Does the check support automatically repair routine? + # + # @return [Boolean] whether check implemented `#repair!` method or not + def can_repair? + self.class.instance_methods(false).include?(:repair!) + end + + def can_skip? + self.class.instance_methods(false).include?(:skip?) + end + + def is_multi_check? + self.class.instance_methods(false).include?(:multi_check) + end + + # Execute the check routine + # + # This is where you should implement the main logic that will return + # a boolean at the end + # + # You should not print any output to STDOUT here, use the specific methods instead + # + # @return [Boolean] whether check passed or failed + def check? + raise NotImplementedError + end + + # Execute a custom check that cover multiple unities + # + # When using multi_check you have to provide the output yourself + def multi_check + raise NotImplementedError + end + + # Prints troubleshooting instructions + # + # This is where you should print detailed information for any error found during #check? + # + # You may use helper methods to help format the output: + # + # @see #try_fixing_it + # @see #fix_and_rerun + # @see #for_more_infromation + def show_error + raise NotImplementedError + end + + # When implemented by a subclass, will attempt to fix the issue automatically + def repair! + raise NotImplementedError + end + + # When implemented by a subclass, will evaluate whether check should be skipped or not + # + # @return [Boolean] whether or not this check should be skipped + def skip? + raise NotImplementedError + end + + def self.call_or_return(input) + input.respond_to?(:call) ? input.call : input + end + private_class_method :call_or_return + end +end diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb new file mode 100644 index 00000000000..c42ae4fe4c4 --- /dev/null +++ b/lib/system_check/helpers.rb @@ -0,0 +1,75 @@ +require 'tasks/gitlab/task_helpers' + +module SystemCheck + module Helpers + include ::Gitlab::TaskHelpers + + # Display a message telling to fix and rerun the checks + def fix_and_rerun + $stdout.puts ' Please fix the error above and rerun the checks.'.color(:red) + end + + # Display a formatted list of references (documentation or links) where to find more information + # + # @param [Array<String>] sources one or more references (documentation or links) + def for_more_information(*sources) + $stdout.puts ' For more information see:'.color(:blue) + sources.each do |source| + $stdout.puts " #{source}" + end + end + + def see_installation_guide_section(section) + "doc/install/installation.md in section \"#{section}\"" + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + + # @deprecated This will no longer be used when all checks were executed using SystemCheck + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Display a formatted list of instructions on how to fix the issue identified by the #check? + # + # @param [Array<String>] steps one or short sentences with help how to fix the issue + def try_fixing_it(*steps) + steps = steps.shift if steps.first.is_a?(Array) + + $stdout.puts ' Try fixing it:'.color(:blue) + steps.each do |step| + $stdout.puts " #{step}" + end + end + + def sanitized_message(project) + if should_sanitize? + "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " + else + "#{project.name_with_namespace.color(:yellow)} ... " + end + end + + def should_sanitize? + if ENV['SANITIZE'] == 'true' + true + else + false + end + end + + def omnibus_gitlab? + Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' + end + + def sudo_gitlab(command) + "sudo -u #{gitlab_user} -H #{command}" + end + end +end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb new file mode 100644 index 00000000000..dc2d4643a01 --- /dev/null +++ b/lib/system_check/simple_executor.rb @@ -0,0 +1,99 @@ +module SystemCheck + # Simple Executor is current default executor for GitLab + # It is a simple port from display logic in the old check.rake + # + # There is no concurrency level and the output is progressively + # printed into the STDOUT + # + # @attr_reader [Array<BaseCheck>] checks classes of corresponding checks to be executed in the same order + # @attr_reader [String] component name of the component relative to the checks being executed + class SimpleExecutor + attr_reader :checks + attr_reader :component + + # @param [String] component name of the component relative to the checks being executed + def initialize(component) + raise ArgumentError unless component.is_a? String + + @component = component + @checks = Set.new + end + + # Add a check to be executed + # + # @param [BaseCheck] check class + def <<(check) + raise ArgumentError unless check < BaseCheck + @checks << check + end + + # Executes defined checks in the specified order and outputs confirmation or error information + def execute + start_checking(component) + + @checks.each do |check| + run_check(check) + end + + finished_checking(component) + end + + # Executes a single check + # + # @param [SystemCheck::BaseCheck] check_klass + def run_check(check_klass) + $stdout.print "#{check_klass.display_name} ... " + + check = check_klass.new + + # When implements skip method, we run it first, and if true, skip the check + if check.can_skip? && check.skip? + $stdout.puts check_klass.skip_reason.color(:magenta) + return + end + + # When implements a multi check, we don't control the output + if check.is_multi_check? + check.multi_check + return + end + + if check.check? + $stdout.puts check_klass.check_pass.color(:green) + else + $stdout.puts check_klass.check_fail.color(:red) + + if check.can_repair? + $stdout.print 'Trying to fix error automatically. ...' + if check.repair! + $stdout.puts 'Success'.color(:green) + return + else + $stdout.puts 'Failed'.color(:red) + end + end + + check.show_error + end + end + + private + + # Prints header content for the series of checks to be executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def start_checking(component) + $stdout.puts "Checking #{component.color(:yellow)} ..." + $stdout.puts '' + end + + # Prints footer content for the series of checks executed for this component + # + # @param [String] component name of the component relative to the checks being executed + def finished_checking(component) + $stdout.puts '' + $stdout.puts "Checking #{component.color(:yellow)} ... #{'Finished'.color(:green)}" + $stdout.puts '' + end + end +end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index b5572a39d30..87ca39b079b 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -21,7 +21,7 @@ namespace :gemojione do moji: emoji_hash['moji'], description: emoji_hash['description'], unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), - digest: hash_digest, + digest: hash_digest } resultant_emoji_map[name] = entry diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 0aa21a4bd13..b27f7475115 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -11,4 +11,12 @@ namespace :gettext do "{#{folders}}/**/*.{#{exts}}" ) end + + task :compile do + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 + FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) + + Rake::Task['gettext:pack'].invoke + Rake::Task['gettext:po_to_json'].invoke + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index f41c73154f5..63c5e9b9c83 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,5 +1,9 @@ +# Temporary hack, until we migrate all checks to SystemCheck format +require 'system_check' +require 'system_check/helpers' + namespace :gitlab do - desc "GitLab | Check the configuration of GitLab and its environment" + desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check gitlab:incoming_email:check @@ -7,331 +11,38 @@ namespace :gitlab do gitlab:app:check} namespace :app do - desc "GitLab | Check the configuration of the GitLab Rails app" + desc 'GitLab | Check the configuration of the GitLab Rails app' task check: :environment do warn_user_is_not_gitlab - start_checking "GitLab" - - check_git_config - check_database_config_exists - check_migrations_are_up - check_orphaned_group_members - check_gitlab_config_exists - check_gitlab_config_not_outdated - check_log_writable - check_tmp_writable - check_uploads - check_init_script_exists - check_init_script_up_to_date - check_projects_have_namespace - check_redis_version - check_ruby_version - check_git_version - check_active_users - - finished_checking "GitLab" - end - - # Checks - ######################## - - def check_git_config - print "Git configured with autocrlf=input? ... " - - options = { - "core.autocrlf" => "input" - } - - correct_options = options.map do |name, value| - run_command(%W(#{Gitlab.config.git.bin_path} config --global --get #{name})).try(:squish) == value - end - - if correct_options.all? - puts "yes".color(:green) - else - print "Trying to fix Git error automatically. ..." - - if auto_fix_git_config(options) - puts "Success".color(:green) - else - puts "Failed".color(:red) - try_fixing_it( - sudo_gitlab("\"#{Gitlab.config.git.bin_path}\" config --global core.autocrlf \"#{options["core.autocrlf"]}\"") - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - end - end - end - - def check_database_config_exists - print "Database config exists? ... " - - database_config_file = Rails.root.join("config", "database.yml") - - if File.exist?(database_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/database.yml.<your db> to config/database.yml", - "Check that the information in config/database.yml is correct" - ) - for_more_information( - see_database_guide, - "http://guides.rubyonrails.org/getting_started.html#configuring-a-database" - ) - fix_and_rerun - end - end - - def check_gitlab_config_exists - print "GitLab config exists? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - - if File.exist?(gitlab_config_file) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_gitlab_config_not_outdated - print "GitLab config outdated? ... " - - gitlab_config_file = Rails.root.join("config", "gitlab.yml") - unless File.exist?(gitlab_config_file) - puts "can't check because of previous errors".color(:magenta) - end - - # omniauth or ldap could have been deleted from the file - unless Gitlab.config['git_host'] - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Backup your config/gitlab.yml", - "Copy config/gitlab.yml.example to config/gitlab.yml", - "Update config/gitlab.yml to match your setup" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_init_script_exists - print "Init script exists? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - script_path = "/etc/init.d/gitlab" - - if File.exist?(script_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Install the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_init_script_up_to_date - print "Init script up-to-date? ... " - - if omnibus_gitlab? - puts 'skipped (omnibus-gitlab has no init script)'.color(:magenta) - return - end - - recipe_path = Rails.root.join("lib/support/init.d/", "gitlab") - script_path = "/etc/init.d/gitlab" - - unless File.exist?(script_path) - puts "can't check because of previous errors".color(:magenta) - return - end - - recipe_content = File.read(recipe_path) - script_content = File.read(script_path) - - if recipe_content == script_content - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Redownload the init script" - ) - for_more_information( - see_installation_guide_section "Install Init Script" - ) - fix_and_rerun - end - end - - def check_migrations_are_up - print "All migrations up? ... " - - migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) - - unless migration_status =~ /down\s+\d{14}/ - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - sudo_gitlab("bundle exec rake db:migrate RAILS_ENV=production") - ) - fix_and_rerun - end - end - - def check_orphaned_group_members - print "Database contains orphaned GroupMembers? ... " - if GroupMember.where("user_id not in (select id from users)").count > 0 - puts "yes".color(:red) - try_fixing_it( - "You can delete the orphaned records using something along the lines of:", - sudo_gitlab("bundle exec rails runner -e production 'GroupMember.where(\"user_id NOT IN (SELECT id FROM users)\").delete_all'") - ) - else - puts "no".color(:green) - end - end - - def check_log_writable - print "Log directory writable? ... " - - log_path = Rails.root.join("log") - - if File.writable?(log_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{log_path}", - "sudo chmod -R u+rwX #{log_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - def check_tmp_writable - print "Tmp directory writable? ... " - - tmp_path = Rails.root.join("tmp") - - if File.writable?(tmp_path) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R gitlab #{tmp_path}", - "sudo chmod -R u+rwX #{tmp_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_uploads - print "Uploads directory setup correctly? ... " - - unless File.directory?(Rails.root.join('public/uploads')) - puts "no".color(:red) - try_fixing_it( - "sudo -u #{gitlab_user} mkdir #{Rails.root}/public/uploads" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - return - end - - upload_path = File.realpath(Rails.root.join('public/uploads')) - upload_path_tmp = File.join(upload_path, 'tmp') - - if File.stat(upload_path).mode == 040700 - unless Dir.exist?(upload_path_tmp) - puts 'skipped (no tmp uploads folder yet)'.color(:magenta) - return - end - - # If tmp upload dir has incorrect permissions, assume others do as well - # Verify drwx------ permissions - if File.stat(upload_path_tmp).mode == 040700 && File.owned?(upload_path_tmp) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chown -R #{gitlab_user} #{upload_path}", - "sudo find #{upload_path} -type f -exec chmod 0644 {} \\;", - "sudo find #{upload_path} -type d -not -path #{upload_path} -exec chmod 0700 {} \\;" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod 700 #{upload_path}" - ) - for_more_information( - see_installation_guide_section "GitLab" - ) - fix_and_rerun - end - end - - def check_redis_version - min_redis_version = "2.8.0" - print "Redis version >= #{min_redis_version}? ... " - - redis_version = run_command(%w(redis-cli --version)) - redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) - if redis_version && - (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your redis server to a version >= #{min_redis_version}" - ) - for_more_information( - "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq" - ) - fix_and_rerun - end + checks = [ + SystemCheck::App::GitConfigCheck, + SystemCheck::App::DatabaseConfigExistsCheck, + SystemCheck::App::MigrationsAreUpCheck, + SystemCheck::App::OrphanedGroupMembersCheck, + SystemCheck::App::GitlabConfigExistsCheck, + SystemCheck::App::GitlabConfigUpToDateCheck, + SystemCheck::App::LogWritableCheck, + SystemCheck::App::TmpWritableCheck, + SystemCheck::App::UploadsDirectoryExistsCheck, + SystemCheck::App::UploadsPathPermissionCheck, + SystemCheck::App::UploadsPathTmpPermissionCheck, + SystemCheck::App::InitScriptExistsCheck, + SystemCheck::App::InitScriptUpToDateCheck, + SystemCheck::App::ProjectsHaveNamespaceCheck, + SystemCheck::App::RedisVersionCheck, + SystemCheck::App::RubyVersionCheck, + SystemCheck::App::GitVersionCheck, + SystemCheck::App::ActiveUsersCheck + ] + + SystemCheck.run('GitLab', checks) end end namespace :gitlab_shell do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of GitLab Shell" task check: :environment do warn_user_is_not_gitlab @@ -513,33 +224,6 @@ namespace :gitlab do end end - def check_projects_have_namespace - print "projects have namespace: ... " - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - - if project.namespace - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Migrate global projects" - ) - for_more_information( - "doc/update/5.4-to-6.0.md in section \"#global-projects\"" - ) - fix_and_rerun - end - end - end - # Helper methods ######################## @@ -565,6 +249,8 @@ namespace :gitlab do end namespace :sidekiq do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Sidekiq" task check: :environment do warn_user_is_not_gitlab @@ -623,6 +309,8 @@ namespace :gitlab do end namespace :incoming_email do + include SystemCheck::Helpers + desc "GitLab | Check the configuration of Reply by email" task check: :environment do warn_user_is_not_gitlab @@ -757,6 +445,8 @@ namespace :gitlab do end namespace :ldap do + include SystemCheck::Helpers + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. @@ -812,6 +502,8 @@ namespace :gitlab do end namespace :repo do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do Gitlab.config.repositories.storages.each do |name, repository_storage| @@ -826,6 +518,8 @@ namespace :gitlab do end namespace :user do + include SystemCheck::Helpers + desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) @@ -848,55 +542,6 @@ namespace :gitlab do # Helper methods ########################## - def fix_and_rerun - puts " Please fix the error above and rerun the checks.".color(:red) - end - - def for_more_information(*sources) - sources = sources.shift if sources.first.is_a?(Array) - - puts " For more information see:".color(:blue) - sources.each do |source| - puts " #{source}" - end - end - - def finished_checking(component) - puts "" - puts "Checking #{component.color(:yellow)} ... #{"Finished".color(:green)}" - puts "" - end - - def see_database_guide - "doc/install/databases.md" - end - - def see_installation_guide_section(section) - "doc/install/installation.md in section \"#{section}\"" - end - - def sudo_gitlab(command) - "sudo -u #{gitlab_user} -H #{command}" - end - - def gitlab_user - Gitlab.config.gitlab.user - end - - def start_checking(component) - puts "Checking #{component.color(:yellow)} ..." - puts "" - end - - def try_fixing_it(*steps) - steps = steps.shift if steps.first.is_a?(Array) - - puts " Try fixing it:".color(:blue) - steps.each do |step| - puts " #{step}" - end - end - def check_gitlab_shell required_version = Gitlab::VersionInfo.new(gitlab_shell_major_version, gitlab_shell_minor_version, gitlab_shell_patch_version) current_version = Gitlab::VersionInfo.parse(gitlab_shell_version) @@ -909,65 +554,6 @@ namespace :gitlab do end end - def check_ruby_version - required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) - - print "Ruby version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your ruby to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_git_version - required_version = Gitlab::VersionInfo.new(2, 7, 3) - current_version = Gitlab::VersionInfo.parse(run_command(%W(#{Gitlab.config.git.bin_path} --version))) - - puts "Your git bin path is \"#{Gitlab.config.git.bin_path}\"" - print "Git version >= #{required_version} ? ... " - - if current_version.valid? && required_version <= current_version - puts "yes (#{current_version})".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "Update your git to a version >= #{required_version} from #{current_version}" - ) - fix_and_rerun - end - end - - def check_active_users - puts "Active users: #{User.active.count}" - end - - def omnibus_gitlab? - Dir.pwd == '/opt/gitlab/embedded/service/gitlab-rails' - end - - def sanitized_message(project) - if should_sanitize? - "#{project.namespace_id.to_s.color(:yellow)}/#{project.id.to_s.color(:yellow)} ... " - else - "#{project.name_with_namespace.color(:yellow)} ... " - end - end - - def should_sanitize? - if ENV['SANITIZE'] == "true" - true - else - false - end - end - def check_repo_integrity(repo_dir) puts "\nChecking repo at #{repo_dir.color(:yellow)}" diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index a2a2db487b7..e3883278886 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -16,6 +16,8 @@ namespace :gitlab do redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a # check Git version git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a + # check Go version + go_version = run_and_match(%w(go version), /go version (.+)/).to_a puts "" puts "System information".color(:yellow) @@ -30,6 +32,7 @@ namespace :gitlab do puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" + puts "Go Version:\t#{go_version[1] || "unknown".color(:red)}" # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e3c9d3b491c..964aa0fe1bc 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -98,34 +98,30 @@ module Gitlab end end + def gitlab_user + Gitlab.config.gitlab.user + end + + def is_gitlab_user? + return @is_gitlab_user unless @is_gitlab_user.nil? + + current_user = run_command(%w(whoami)).chomp + @is_gitlab_user = current_user == gitlab_user + end + def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user + return if @warned_user_not_gitlab + + unless is_gitlab_user? current_user = run_command(%w(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" - command_success.all? - else - false + @warned_user_not_gitlab = true end end diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index fc0ccc726ed..7728c485e8d 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -19,5 +19,21 @@ namespace :gitlab do puts "There are currently no users with 2FA enabled.".color(:yellow) end end + + namespace :rotate_key do + def rotator + @rotator ||= Gitlab::OtpKeyRotator.new(ENV['filename']) + end + + desc "Encrypt user OTP secrets with a new encryption key" + task apply: :environment do |t, args| + rotator.rotate!(old_key: ENV['old_key'], new_key: ENV['new_key']) + end + + desc "Rollback to secrets encrypted with the old encryption key" + task rollback: :environment do + rotator.rollback! + end + end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 1b04e1350ed..59c32bbe7a4 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -49,7 +49,7 @@ namespace :gitlab do Template.new( "https://gitlab.com/gitlab-org/Dockerfile.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ - ), + ) ].freeze def vendor_directory diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index bc76d7edc55..50b8e331469 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -37,7 +37,7 @@ class GithubImport end def import! - @project.import_start + @project.force_import_start timings = Benchmark.measure do Github::Import.new(@project, @options).execute diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 602c60be828..2eddcb3c777 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -60,7 +60,7 @@ desc "GitLab | Run specs" task :spec do cmds = [ %w(rake gitlab:setup), - %w(rspec spec), + %w(rspec spec) ] run_commands(cmds) end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 95735f43802..ad1818ff1fa 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -11,6 +11,11 @@ namespace :tokens do reset_all_users_token(:reset_incoming_email_token!) end + desc "Reset all GitLab RSS tokens" + task reset_all_rss: :environment do + reset_all_users_token(:reset_rss_token!) + end + def reset_all_users_token(reset_token_method) TmpUser.find_in_batches do |batch| puts "Processing batch starting with user ID: #{batch.first.id}" @@ -35,4 +40,9 @@ class TmpUser < ActiveRecord::Base write_new_token(:incoming_email_token) save!(validate: false) end + + def reset_rss_token! + write_new_token(:rss_token) + save!(validate: false) + end end |