diff options
Diffstat (limited to 'lib')
162 files changed, 1829 insertions, 968 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index bf8ddba6f0d..f4a96b9711b 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -6,7 +6,7 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") - NO_SLASH_URL_PART_REGEX = %r{[^/]+} + NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze @@ -134,6 +134,7 @@ module API mount ::API::Pipelines mount ::API::PipelineSchedules mount ::API::ProjectClusters + mount ::API::ProjectEvents mount ::API::ProjectExport mount ::API::ProjectImport mount ::API::ProjectHooks diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 5c98b0ad56c..65d7f68bbf9 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -162,8 +162,8 @@ module API result = DeleteBranchService.new(user_project, current_user) .execute(params[:branch]) - if result[:status] != :success - render_api_error!(result[:message], result[:return_code]) + if result.error? + render_api_error!(result.message, result.http_status) end end end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 8afe6dda414..5928ee1657b 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -134,9 +134,13 @@ module API post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do noteable = find_noteable(parent_type, noteables_str, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) + first_note = notes.first break not_found!("Discussion") if notes.empty? - break bad_request!("Discussion is an individual note.") unless notes.first.part_of_discussion? + + unless first_note.part_of_discussion? || first_note.to_discussion.can_convert_to_discussion? + break bad_request!("Discussion can not be replied to.") + end opts = { note: params[:body], diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4bdac278add..90ed24a2ded 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -922,7 +922,15 @@ module API end class NamespaceBasic < Grape::Entity - expose :id, :name, :path, :kind, :full_path, :parent_id + expose :id, :name, :path, :kind, :full_path, :parent_id, :avatar_url + + expose :web_url do |namespace| + if namespace.user? + Gitlab::Routing.url_helpers.user_url(namespace.owner) + else + namespace.web_url + end + end end class Namespace < NamespaceBasic @@ -1148,22 +1156,33 @@ module API end end - class Release < TagRelease + class Release < Grape::Entity expose :name + expose :tag, as: :tag_name, if: lambda { |_, _| can_download_code? } + expose :description expose :description_html do |entity| MarkupHelper.markdown_field(entity, :description) end expose :created_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } - expose :commit, using: Entities::Commit + expose :commit, using: Entities::Commit, if: lambda { |_, _| can_download_code? } expose :assets do - expose :assets_count, as: :count - expose :sources, using: Entities::Releases::Source + expose :assets_count, as: :count do |release, _| + assets_to_exclude = can_download_code? ? [] : [:sources] + release.assets_count(except: assets_to_exclude) + end + expose :sources, using: Entities::Releases::Source, if: lambda { |_, _| can_download_code? } expose :links, using: Entities::Releases::Link do |release, options| release.links.sorted end end + + private + + def can_download_code? + Ability.allowed?(options[:current_user], :download_code, object.project) + end end class Tag < Grape::Entity @@ -1269,7 +1288,7 @@ module API end class Variable < Grape::Entity - expose :key, :value + expose :variable_type, :key, :value expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) } end diff --git a/lib/api/events.rb b/lib/api/events.rb index b98aa9f31e1..e4c017fab42 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -4,34 +4,11 @@ module API class Events < Grape::API include PaginationParams include APIGuard + helpers ::API::Helpers::EventsHelpers - helpers do - params :event_filter_params do - optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' - optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' - optional :before, type: Date, desc: 'Include only events created before this date' - optional :after, type: Date, desc: 'Include only events created after this date' - end - - params :sort_params do - optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return events sorted in ascending and descending order' - end - - def present_events(events) - events = paginate(events) - - present events, with: Entities::Event - end - - def find_events(source) - EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute - end - end + allow_access_with_scope :read_user, if: -> (request) { request.get? } resource :events do - allow_access_with_scope :read_user, if: -> (request) { request.get? } - desc "List currently authenticated user's events" do detail 'This feature was introduced in GitLab 9.3.' success Entities::Event @@ -55,8 +32,6 @@ module API requires :id, type: String, desc: 'The ID or Username of the user' end resource :users do - allow_access_with_scope :read_user, if: -> (request) { request.get? } - desc 'Get the contribution events of a specified user' do detail 'This feature was introduced in GitLab 8.13.' success Entities::Event @@ -76,25 +51,5 @@ module API present_events(events) end end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "List a Project's visible events" do - success Entities::Event - end - params do - use :pagination - use :event_filter_params - use :sort_params - end - - get ":id/events" do - events = find_events(user_project) - - present_events(events) - end - end end end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 3f048e0dc56..47fcbabb4d4 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -47,6 +47,7 @@ module API requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' end post ':id/variables' do variable_params = declared_params(include_missing: false) @@ -67,6 +68,7 @@ module API optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 8a21d44b4bf..7e4539d0419 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -67,10 +67,6 @@ module API initial_current_user != current_user end - def user_namespace - @user_namespace ||= find_namespace!(params[:id]) - end - def user_group @group ||= find_group!(params[:id]) end diff --git a/lib/api/helpers/events_helpers.rb b/lib/api/helpers/events_helpers.rb new file mode 100644 index 00000000000..bf3b76bb92d --- /dev/null +++ b/lib/api/helpers/events_helpers.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module API + module Helpers + module EventsHelpers + extend Grape::API::Helpers + + params :event_filter_params do + optional :action, type: String, values: Event.actions, desc: 'Event action to filter on' + optional :target_type, type: String, values: Event.target_types, desc: 'Event target type to filter on' + optional :before, type: Date, desc: 'Include only events created before this date' + optional :after, type: Date, desc: 'Include only events created after this date' + end + + params :sort_params do + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return events sorted in ascending and descending order' + end + + def present_events(events) + events = paginate(events) + + present events, with: Entities::Event + end + + def find_events(source) + EventsFinder.new(params.merge(source: source, current_user: current_user, with_associations: true)).execute + end + end + end +end diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index 793ae11b41d..9cdde25fe4e 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -13,6 +13,10 @@ module API available?(:merge_requests, project, options[:current_user]) end + def expose_path(path) + Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, path) + end + def expose_url(path) url_options = Gitlab::Application.routes.default_url_options protocol, host, port, script_name = url_options.values_at(:protocol, :host, :port, :script_name) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 00f0bbab231..c82fd230d7a 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -265,6 +265,8 @@ module API params[:changes], push_options.as_json) if Feature.enabled?(:mr_push_options, default_enabled: true) + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/61359') + mr_options = push_options.get(:merge_request) output.merge!(process_mr_push_options(mr_options, project, user, params[:changes])) if mr_options.present? end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 000c00ea9f9..d0a93b77951 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -192,6 +192,7 @@ module API params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project) issue_params = declared_params(include_missing: false) + issue_params[:system_note_timestamp] = params[:created_at] issue_params = convert_parameters_from_legacy_format(issue_params) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index ce85772e4ed..daa98c22e5e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -367,6 +367,10 @@ module API merge_request = find_project_merge_request(params[:merge_request_iid]) merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) + if merge_when_pipeline_succeeds || merge_request.merge_when_pipeline_succeeds + render_api_error!('Not allowed: pipeline does not exist', 405) unless merge_request.head_pipeline + end + # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 3cc09f6ac3f..77ecb3e7cde 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -44,6 +44,8 @@ module API requires :id, type: String, desc: "Namespace's ID or path" end get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + user_namespace = find_namespace!(params[:id]) + present user_namespace, with: Entities::Namespace, current_user: current_user end end diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index c86b50d3736..1d1ef1afc6b 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -118,6 +118,7 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' + optional :variable_type, type: String, values: Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' end post ':id/pipeline_schedules/:pipeline_schedule_id/variables' do authorize! :update_pipeline_schedule, pipeline_schedule @@ -138,6 +139,7 @@ module API requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id' requires :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' + optional :variable_type, type: String, values: Ci::PipelineScheduleVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' end put ':id/pipeline_schedules/:pipeline_schedule_id/variables/:key' do authorize! :update_pipeline_schedule, pipeline_schedule diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index b62ec887183..dcc8d94fb79 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -54,6 +54,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :domain, type: String, desc: 'Cluster base domain' + optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do requires :api_url, type: String, allow_blank: false, desc: 'URL to access the Kubernetes API' requires :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb new file mode 100644 index 00000000000..734311e1142 --- /dev/null +++ b/lib/api/project_events.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + class ProjectEvents < Grape::API + include PaginationParams + include APIGuard + helpers ::API::Helpers::EventsHelpers + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "List a Project's visible events" do + success Entities::Event + end + params do + use :pagination + use :event_filter_params + use :sort_params + end + + get ":id/events" do + events = find_events(user_project) + + present_events(events) + end + end + end +end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index cb85028f22c..6b17f4317db 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -23,7 +23,7 @@ module API get ':id/releases' do releases = ::ReleasesFinder.new(user_project, current_user).execute - present paginate(releases), with: Entities::Release + present paginate(releases), with: Entities::Release, current_user: current_user end desc 'Get a single project release' do @@ -34,9 +34,9 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag end get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMETS do - authorize_read_release! + authorize_download_code! - present release, with: Entities::Release + present release, with: Entities::Release, current_user: current_user end desc 'Create a new release' do @@ -63,7 +63,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -86,7 +86,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -107,7 +107,7 @@ module API .execute if result[:status] == :success - present result[:release], with: Entities::Release + present result[:release], with: Entities::Release, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end @@ -135,6 +135,10 @@ module API authorize! :destroy_release, release end + def authorize_download_code! + authorize! :download_code, release + end + def release @release ||= user_project.releases.find_by_tag(params[:tag]) end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index c60d25b88cb..ea36c24eca2 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -15,12 +15,14 @@ module API optional :info, type: Hash, desc: %q(Runner's metadata) optional :active, type: Boolean, desc: 'Should Runner be active' optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' + optional :access_level, type: String, values: Ci::Runner.access_levels.keys, + desc: 'The access_level of the runner' optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs' optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end post '/' do - attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :maximum_timeout]) + attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :access_level, :maximum_timeout]) .merge(get_runner_details_from_request) attributes = diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b064747e5fc..8046acfa397 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -150,6 +150,12 @@ module API given elasticsearch_indexing: ->(val) { val } do optional :elasticsearch_search, type: Boolean, desc: 'Enable Elasticsearch search' requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")' + optional :elasticsearch_limit_indexing, type: Boolean, desc: 'Limit Elasticsearch to index certain namespaces and projects' + end + + given elasticsearch_limit_indexing: ->(val) { val } do + optional :elasticsearch_namespace_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The namespace ids to index with Elasticsearch.' + optional :elasticsearch_project_ids, type: Array[Integer], coerce_with: Validations::Types::LabelsList.coerce, desc: 'The project ids to index with Elasticsearch.' end optional :email_additional_text, type: String, desc: 'Additional text added to the bottom of every email for legal/auditing/compliance reasons' diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 3489ba827e4..a1bb21b3a06 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -55,6 +55,7 @@ module API requires :key, type: String, desc: 'The key of the variable' requires :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' if Gitlab.ee? optional :environment_scope, type: String, desc: 'The environment_scope of the variable' @@ -80,6 +81,7 @@ module API optional :key, type: String, desc: 'The key of the variable' optional :value, type: String, desc: 'The value of the variable' optional :protected, type: String, desc: 'Whether the variable is protected' + optional :variable_type, type: String, values: Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' if Gitlab.ee? optional :environment_scope, type: String, desc: 'The environment_scope of the variable' diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 994074ddc67..5724adb2c40 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -33,7 +33,8 @@ module API authorize! :read_wiki, user_project entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic - present user_project.wiki.pages, with: entity + + present user_project.wiki.list_pages(load_content: params[:with_content]), with: entity end desc 'Get a wiki page' do diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index 6d01d51955c..cce79e73d2d 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -2,13 +2,13 @@ module Banzai module ColorParser - ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 - PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% - ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ - BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 - DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] - RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad - HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/.freeze # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/.freeze # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/.freeze + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/.freeze # 00..255 + DEGS = /-?\d+(?:deg)?/i.freeze # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i.freeze # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/.freeze RGB_FORMAT = %r{ (?:rgba? \( @@ -20,7 +20,7 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze HSL_FORMAT = %r{ (?:hsla? \( @@ -28,11 +28,11 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi + }xi.freeze FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze - COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix.freeze # Public: Analyzes whether the String is a color code. # diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 086adf59d2b..56214043d87 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -33,7 +33,7 @@ module Banzai # https://github.com/vmg/rinku/blob/v2.0.1/ext/rinku/autolink.c#L65 # # Rubular: http://rubular.com/r/nrL3r9yUiq - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)}.freeze # Text matching LINK_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index a27d18facd1..544231adea4 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -20,7 +20,7 @@ module Banzai \s* ^\k<delim> # closing front matter marker \s* - }mx + }mx.freeze def call html.sub(PATTERN) do |_match| diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 77e4c438bd0..4d67140b0a1 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -82,16 +82,18 @@ module Banzai def object_link_text(object, matches) label_suffix = '' + parent = project || group if project || full_path_ref?(matches) project_path = full_project_path(matches[:namespace], matches[:project]) parent_from_ref = from_ref_cached(project_path) - reference = parent_from_ref.to_human_reference(project || group) + reference = parent_from_ref.to_human_reference(parent) label_suffix = " <i>in #{reference}</i>" if reference.present? end - LabelsHelper.render_colored_label(object, label_suffix: label_suffix, title: tooltip_title(object)) + presenter = object.present(issuable_subject: parent) + LabelsHelper.render_colored_label(presenter, label_suffix: label_suffix, title: tooltip_title(presenter)) end def tooltip_title(label) diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 50bf823929c..ee7f10ebdf6 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -33,7 +33,7 @@ module Banzai (?<new_link>.+?) (?<title>\ ".+?")? \) - }x + }x.freeze # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index f2ae17b44fa..ade4d260be1 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -17,7 +17,7 @@ module Banzai # :toc - String containing Table of Contents data as a `ul` element with # `li` child elements. class TableOfContentsFilter < HTML::Pipeline::Filter - PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u + PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze def call return doc if context[:no_header_anchors] @@ -31,6 +31,7 @@ module Banzai if header_content = node.children.first id = node .text + .strip .downcase .gsub(PUNCTUATION_REGEXP, '') # remove punctuation .tr(' ', '-') # replace spaces with dash diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index 239780d8626..9b7d1548056 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module DeclarativePolicy diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 1204e53ee2e..3f107fbbf3b 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -36,8 +36,8 @@ module Gitlab end COM_URL = 'https://gitlab.com'.freeze - APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} - SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z} + APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze + SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze @@ -59,7 +59,11 @@ module Gitlab end def self.ee? - Object.const_defined?(:License) + if ENV['IS_GITLAB_EE'].present? + Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) + else + Object.const_defined?(:License) + end end def self.process_name diff --git a/lib/gitlab/action_view_output/context.rb b/lib/gitlab/action_view_output/context.rb new file mode 100644 index 00000000000..9fbc9811636 --- /dev/null +++ b/lib/gitlab/action_view_output/context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# This file was simplified from https://raw.githubusercontent.com/rails/rails/195f39804a7a4a0034f25e8704220e03d95a752a/actionview/lib/action_view/context.rb. +# +# It is only needed by modules that need to call ActionView helper +# methods (e.g. those in +# https://github.com/rails/rails/tree/c4d3e202e10ae627b3b9c34498afb45450652421/actionview/lib/action_view/helpers) +# to generate tags outside of a Rails controller (e.g. API, Sidekiq, +# etc.). +# +# In Rails 5, ActionView::Context automatically includes CompiledTemplates. +# This means that any module that includes ActionView::Context is now a descendant +# of CompiledTemplates. +# +# When a partial is rendered for the first time, it runs +# Module#module_eval, which will evaluate a string source that defines a +# new method. For example: +# +# def _app_views_profiles_show_html_haml___1285955918103175884_70307801785400(local_assigns, output_buffer) +# "hello world" +# end +# +# When a new method is defined, the Ruby interpreter clears the method +# cache for all descendants, and all methods for those modules will have +# to be redefined. This can lead to a significant performance penalty. +# +# Rails 6 fixes this behavior by moving out the `include +# CompiledTemplates` into ActionView::Base so that including `ActionView::Context` +# doesn't quietly affect other modules in this way. + +if Rails::VERSION::STRING.start_with?('6') + raise 'This module is no longer needed in Rails 6. Use ActionView::Context instead.' +end + +module Gitlab + module ActionViewOutput + module Context + attr_accessor :output_buffer, :view_flow + end + end +end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 78fa25c5516..4ebf2afb9cb 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb index a84f794bfae..1924f2ffee2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb @@ -8,8 +8,8 @@ module Gitlab self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze # These regex patterns are tested against a relative path, relative to # the upload directory. diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 81ca2b0a9b7..1ee44a3a5a9 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -16,7 +16,7 @@ module Gitlab RELATIVE_UPLOAD_DIR ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 1d3ddeeb0f1..ff2694abd5e 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -201,6 +201,7 @@ module Gitlab target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), target_branch_sha: pull_request.target_branch_sha, state: pull_request.state, + state_id: MergeRequest.available_states[pull_request.state], author_id: author_id, assignee_id: nil, created_at: pull_request.created_at, diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index 1dbd564fb6f..4ddc1c718c7 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -48,7 +48,7 @@ module Gitlab if project.empty_repo? protected_branch_push_checks - elsif creation? && protected_branch_creation_enabled? + elsif creation? protected_branch_creation_checks elsif deletion? protected_branch_deletion_checks @@ -124,10 +124,6 @@ module Gitlab Gitlab::Routing.url_helpers.project_project_members_url(project) end - def protected_branch_creation_enabled? - Feature.enabled?(:protected_branch_creation, project, default_enabled: true) - end - def matching_merge_request? Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index cc6a14d2d9a..67a65d61441 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,6 +7,7 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'.freeze def validate! + return unless Feature.enabled?(:lfs_check, default_enabled: true) return unless project.lfs_enabled? return if skip_lfs_integrity_check diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 7011dd1aaf2..1c3ce08be76 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -11,8 +11,8 @@ module Gitlab ParserError = Class.new(StandardError) InvalidStreamError = Class.new(StandardError) - VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ - INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/.freeze + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}.freeze attr_reader :stream, :path, :full_version diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index 41135ae62bb..dbdc59505ac 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -6,7 +6,10 @@ module Gitlab module Prerequisite class KubernetesNamespace < Base def unmet? - deployment_cluster.present? && kubernetes_namespace.new_record? + deployment_cluster.present? && + deployment_cluster.managed? && + !deployment_cluster.project_type? && + kubernetes_namespace.new_record? end def complete! diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index a3f6cc31321..eeaac52eefd 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -11,7 +11,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management].freeze + ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management metrics].freeze attributes ALLOWED_KEYS @@ -28,6 +28,7 @@ module Gitlab validates :dast, array_of_strings_or_string: true validates :performance, array_of_strings_or_string: true validates :license_management, array_of_strings_or_string: true + validates :metrics, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb index 1dd2d42016a..4a049ecae49 100644 --- a/lib/gitlab/ci/config/external/processor.rb +++ b/lib/gitlab/ci/config/external/processor.rb @@ -11,7 +11,8 @@ module Gitlab @values = values @external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process @content = {} - rescue External::Mapper::Error => e + rescue External::Mapper::Error, + OpenSSL::SSL::SSLError => e raise IncludeError, e.message end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 03af99ba9a5..c911bfa7ff6 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 7d6e0704d4a..df92e229f12 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -7,7 +7,7 @@ module Gitlab class Skip < Chain::Base include ::Gitlab::Utils::StrongMemoize - SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i.freeze def perform! if skipped? diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb new file mode 100644 index 00000000000..5fcc9406cc8 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class NotEquals < Lexeme::Operator + PATTERN = /!=/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + @left.evaluate(variables) != @right.evaluate(variables) + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb new file mode 100644 index 00000000000..14544d33e25 --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class NotMatches < Lexeme::Operator + PATTERN = /\!~/.freeze + + def initialize(left, right) + @left = left + @right = right + end + + def evaluate(variables = {}) + text = @left.evaluate(variables) + regexp = @right.evaluate(variables) + + regexp.scan(text.to_s).none? + end + + def self.build(_value, behind, ahead) + new(behind, ahead) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index f26542361a2..e14edfae51d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -15,7 +15,9 @@ module Gitlab Expression::Lexeme::Pattern, Expression::Lexeme::Null, Expression::Lexeme::Equals, - Expression::Lexeme::Matches + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches ].freeze MAX_TOKENS = 100 diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index b03611f756e..ab5ae9caeea 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -8,13 +8,24 @@ module Gitlab StatementError = Class.new(Expression::ExpressionError) GRAMMAR = [ + # presence matchers %w[variable], + + # positive matchers %w[variable equals string], %w[variable equals variable], %w[variable equals null], %w[string equals variable], %w[null equals variable], - %w[variable matches pattern] + %w[variable matches pattern], + + # negative matchers + %w[variable notequals string], + %w[variable notequals variable], + %w[variable notequals null], + %w[string notequals variable], + %w[null notequals variable], + %w[variable notmatches pattern] ].freeze def initialize(statement, variables = {}) diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index d7570fdd3e2..8972498a069 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -14,7 +14,7 @@ module Gitlab end def group - 'failed_with_warnings' + 'failed-with-warnings' end def status_tooltip diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 58f4642510b..e50b0853725 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -6,7 +6,8 @@ module Gitlab module Stage class Factory < Status::Factory def self.extended_statuses - [Status::SuccessWarning] + [[Status::SuccessWarning], + [Status::Stage::PlayManual]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/stage/play_manual.rb b/lib/gitlab/ci/status/stage/play_manual.rb new file mode 100644 index 00000000000..ac3fc0912fa --- /dev/null +++ b/lib/gitlab/ci/status/stage/play_manual.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Stage + class PlayManual < Status::Extended + include Gitlab::Routing + + def action_icon + 'play' + end + + def action_title + 'Play all manual' + end + + def action_path + pipeline = subject.pipeline + + project_stage_play_manual_path(pipeline.project, pipeline, subject.name) + end + + def action_method + :post + end + + def action_button_title + _('Play all manual') + end + + def self.matches?(stage, user) + stage.manual_playable? + end + + def has_action? + true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 6632cd9b143..47623ad945f 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -21,7 +21,7 @@ module Gitlab end def group - 'success_with_warnings' + 'success-with-warnings' end def self.matches?(subject, user) diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 1e9591e113b..9d99d04d263 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -394,6 +394,7 @@ rollout 100%: --set postgresql.postgresDatabase="$POSTGRES_DB" \ --set postgresql.imageTag="$POSTGRES_VERSION" \ --set application.initializeCommand="$DB_INITIALIZE" \ + $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ "$name" \ chart/ @@ -403,6 +404,7 @@ rollout 100%: --wait \ --set application.initializeCommand="" \ --set application.migrateCommand="$DB_MIGRATE" \ + $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ "$name" \ chart/ @@ -432,6 +434,7 @@ rollout 100%: --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ --set postgresql.postgresDatabase="$POSTGRES_DB" \ --set application.migrateCommand="$DB_MIGRATE" \ + $HELM_UPGRADE_EXTRA_ARGS \ --namespace="$KUBE_NAMESPACE" \ "$name" \ chart/ @@ -487,7 +490,7 @@ rollout 100%: fi helm init --client-only - helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} + helm repo add ${AUTO_DEVOPS_CHART_REPOSITORY_NAME:-gitlab} ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} ${AUTO_DEVOPS_CHART_REPOSITORY_USERNAME:+"--username" "$AUTO_DEVOPS_CHART_REPOSITORY_USERNAME"} ${AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD:+"--password" "$AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD"} if [[ ! -d "$auto_chart" ]]; then helm fetch ${auto_chart} --untar fi diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index 02d02250bbf..a9e195370f7 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -1,10 +1,11 @@ # Full project: https://gitlab.com/pages/hexo -image: node:6.10.0 +image: node:10.15.3 pages: script: - - npm install - - ./node_modules/hexo/bin/hexo generate + - npm install hexo-cli -g + - test -e package.json && npm install + - hexo generate artifacts: paths: - public diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index eef361c19e9..324e39c7747 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ container_scanning: DOCKER_SERVICE: docker DOCKER_HOST: tcp://${DOCKER_SERVICE}:2375/ # https://hub.docker.com/r/arminc/clair-local-scan/tags - CLAIR_LOCAL_SCAN_VERSION: v2.0.6 + CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1 allow_failure: true services: - docker:stable-dind diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 2a90cc9a06c..fd7fac5dcab 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -30,6 +30,7 @@ dast: - | function dast_run() { docker run \ + --env DAST_FULL_SCAN_ENABLED \ --env DAST_TARGET_AVAILABILITY_TIMEOUT \ --volume "$PWD:/output" \ --volume /var/run/docker.sock:/var/run/docker.sock \ @@ -46,7 +47,8 @@ dast: --auth-username $DAST_USERNAME \ --auth-password $DAST_PASSWORD \ --auth-username-field $DAST_USERNAME_FIELD \ - --auth-password-field $DAST_PASSWORD_FIELD + --auth-password-field $DAST_PASSWORD_FIELD \ + --auth-exclude-urls $DAST_AUTH_EXCLUDE_URLS else dast_run fi diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 7f80a6e9285..8dd9775c583 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -20,16 +20,27 @@ dependency_scanning: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - | docker run \ - --env DS_ANALYZER_IMAGES \ - --env DS_ANALYZER_IMAGE_PREFIX \ - --env DS_ANALYZER_IMAGE_TAG \ - --env DS_DEFAULT_ANALYZERS \ - --env DEP_SCAN_DISABLE_REMOTE_CHECKS \ - --env DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - --env DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - --env DS_RUN_ANALYZER_TIMEOUT \ + $(propagate_env_vars \ + DS_ANALYZER_IMAGES \ + DS_ANALYZER_IMAGE_PREFIX \ + DS_ANALYZER_IMAGE_TAG \ + DS_DEFAULT_ANALYZERS \ + DS_EXCLUDED_PATHS \ + DEP_SCAN_DISABLE_REMOTE_CHECKS \ + DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + DS_RUN_ANALYZER_TIMEOUT \ + ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index b941e89991e..706692e063b 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -20,18 +20,30 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } - | docker run \ - --env SAST_ANALYZER_IMAGES \ - --env SAST_ANALYZER_IMAGE_PREFIX \ - --env SAST_ANALYZER_IMAGE_TAG \ - --env SAST_DEFAULT_ANALYZERS \ - --env SAST_BRAKEMAN_LEVEL \ - --env SAST_GOSEC_LEVEL \ - --env SAST_FLAWFINDER_LEVEL \ - --env SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - --env SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - --env SAST_RUN_ANALYZER_TIMEOUT \ + $(propagate_env_vars \ + SAST_ANALYZER_IMAGES \ + SAST_ANALYZER_IMAGE_PREFIX \ + SAST_ANALYZER_IMAGE_TAG \ + SAST_DEFAULT_ANALYZERS \ + SAST_EXCLUDED_PATHS \ + SAST_BANDIT_EXCLUDED_PATHS \ + SAST_BRAKEMAN_LEVEL \ + SAST_GOSEC_LEVEL \ + SAST_FLAWFINDER_LEVEL \ + SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ + SAST_RUN_ANALYZER_TIMEOUT \ + ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code diff --git a/lib/gitlab/ci/templates/dotNET-Core.yml b/lib/gitlab/ci/templates/dotNET-Core.yml new file mode 100644 index 00000000000..558ca3d22e1 --- /dev/null +++ b/lib/gitlab/ci/templates/dotNET-Core.yml @@ -0,0 +1,107 @@ +# This is a simple example illustrating how to build and test .NET Core project +# with GitLab Continuous Integration / Continuous Delivery. + +# ### Specify the Docker image +# +# Instead of installing .NET Core SDK manually, a docker image is used +# with already pre-installed .NET Core SDK. +# The 'latest' tag targets the latest available version of .NET Core SDK image. +# If preferred, you can explicitly specify version of .NET Core e.g. using '2.2-sdk' tag. +# +# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet +# Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag +# and the Docker itself: https://opensource.com/resources/what-docker +image: microsoft/dotnet:latest + +# ### Define variables +# +variables: + # 1) Name of directory where restore and build objects are stored. + OBJECTS_DIRECTORY: 'obj' + # 2) Name of directory used for keeping restored dependencies. + NUGET_PACKAGES_DIRECTORY: '.nuget' + # 3) A relative path to the source code from project repository root. + # NOTE: Please edit this path so it matches the structure of your project! + SOURCE_CODE_PATH: '*/*/' + +# ### Define stage list +# +# In this example there are only two stages. +# Initially, the project will be built and then tested. +stages: + - build + - test + +# ### Define global cache rule +# +# Before building the project, all dependencies (e.g. third-party NuGet packages) +# must be restored. Jobs on GitLab.com's Shared Runners are executed on autoscaled machines. +# Each machine is used only once (for security reasons) and after that it is removed. +# What that means is that before every job a dependency restore must be performed +# because restored dependencies are removed along with machines. Fortunately, +# GitLab provides cache mechanism with the aim of keeping restored dependencies +# for other jobs. This example shows how to configure cache to pass over restored +# dependencies for re-use. +# +# With global cache rule, cached dependencies will be downloaded before every job +# and then unpacked to the paths as specified below. +cache: + # Per-stage and per-branch caching. + key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + paths: + # Specify three paths that should be cached: + # + # 1) Main JSON file holding information about package dependency tree, packages versions, + # frameworks etc. It also holds information where to the dependencies were restored. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/project.assets.json' + # 2) Other NuGet and MSBuild related files. Also needed. + - '$SOURCE_CODE_PATH$OBJECTS_DIRECTORY/*.csproj.nuget.*' + # 3) Path to the directory where restored dependencies are kept. + - '$NUGET_PACKAGES_DIRECTORY' + # 'pull-push' policy means that latest cache will be downloaded (if exists) + # before executing the job, and a newer version will be uploaded afterwards. + # Such setting saves time when there are no changes in referenced third-party + # packages. For example if you run a pipeline with changes in your code, + # but with no changes within third-party packages which your project is using, + # then project restore will happen in next to no time as all required dependencies + # will already be there — unzipped from cache. 'pull-push' policy is a default + # cache policy, you do not have to specify it explicitly. + policy: pull-push + +# ### Restore project dependencies +# +# NuGet packages by default are restored to '.nuget/packages' directory +# in the user's home directory. That directory is out of scope of GitLab caching. +# To get around this a custom path can be specified using '--packages <PATH>' option +# for 'dotnet restore' command. In this example a temporary directory is created +# in the root of project repository, so it's content can be cached. +# +# Learn more about GitLab cache: https://docs.gitlab.com/ee/ci/caching/index.html +before_script: + - 'dotnet restore --packages $NUGET_PACKAGES_DIRECTORY' + +build: + stage: build + # ### Build all projects discovered from solution file. + # + # Note: this will fail if you have any projects in your solution that are not + # .NET Core based projects e.g. WCF service, which is based on .NET Framework, + # not .NET Core. In such scenario you will need to build every .NET Core based + # project by explicitly specifying a relative path to the directory + # where it is located e.g. 'dotnet build ./src/ConsoleApp'. + # Only one project path can be passed as a parameter to 'dotnet build' command. + script: + - 'dotnet build --no-restore' + +tests: + stage: test + # ### Run the tests + # + # You can either run tests for all test projects that are defined in your solution + # with 'dotnet test' or run tests only for specific project by specifying + # a relative path to the directory where it is located e.g. 'dotnet test ./test/UnitTests'. + # + # You may want to define separate testing jobs for different types of testing + # e.g. integration tests, unit tests etc. + script: + - 'dotnet test --no-restore' diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index bf5f2a31f0e..dfae260239e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -209,10 +209,7 @@ module Gitlab end def paths - [ - default_path, - deprecated_path - ].compact + [default_path] end def default_directory @@ -227,15 +224,6 @@ module Gitlab File.join(default_directory, "#{job.id}.log") end - def deprecated_path - File.join( - Settings.gitlab_ci.builds_path, - job.created_at.utc.strftime("%Y_%m"), - job.project.ci_id.to_s, - "#{job.id}.log" - ) if job.project&.ci_id - end - def trace_artifact job.job_artifacts_trace end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 833aa75adb5..aab10aef398 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -27,13 +27,9 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - # If the `variable_masking` feature is enabled we expose the `masked` - # attribute, otherwise it's not exposed. - # def to_runner_variable @variable.reject do |hash_key, hash_value| - (hash_key == :file && hash_value == false) || - (hash_key == :masked && !Feature.enabled?(:variable_masking)) + hash_key == :file && hash_value == false end end diff --git a/lib/gitlab/content_disposition.rb b/lib/gitlab/content_disposition.rb index 32207514ce5..ff6154a5b26 100644 --- a/lib/gitlab/content_disposition.rb +++ b/lib/gitlab/content_disposition.rb @@ -22,13 +22,13 @@ module Gitlab end # rubocop:disable Style/VariableInterpolation - TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/ + TRADITIONAL_ESCAPED_CHAR = /[^ A-Za-z0-9!#$+.^_`|~-]/.freeze def ascii_filename 'filename="' + percent_escape(::I18n.transliterate(filename), TRADITIONAL_ESCAPED_CHAR) + '"' end - RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/ + RFC_5987_ESCAPED_CHAR = /[^A-Za-z0-9!#$&+.^_`|~-]/.freeze # rubocop:enable Style/VariableInterpolation def utf8_filename diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d347f3c13a4..3ef19d801b7 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -99,15 +99,13 @@ module Gitlab end CATEGORY_LABELS = { - docs: "~Documentation", + docs: "~Documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. none: "", qa: "~QA" }.freeze - - # rubocop:disable Style/RegexpLiteral CATEGORIES = { - %r{\Adoc/} => :docs, - %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, + %r{\Adoc/} => :none, # To reinstate roulette for documentation, set to `:docs`. + %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\A(ee/)?app/(assets|views)/} => :frontend, %r{\A(ee/)?public/} => :frontend, @@ -148,10 +146,9 @@ module Gitlab # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, - %r{\.(md|txt)\z} => :docs, + %r{\.(md|txt)\z} => :none, # To reinstate roulette for documentation, set to `:docs`. %r{\.js\z} => :frontend }.freeze - # rubocop:enable Style/RegexpLiteral end end end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb new file mode 100644 index 00000000000..f11e032ab84 --- /dev/null +++ b/lib/gitlab/data_builder/deployment.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Deployment + extend self + + def build(deployment) + { + object_kind: 'deployment', + status: deployment.status, + deployable_id: deployment.deployable_id, + deployable_url: Gitlab::UrlBuilder.build(deployment.deployable), + environment: deployment.environment.name, + project: deployment.project.hook_attrs, + short_sha: deployment.short_sha, + user: deployment.user.hook_attrs, + user_url: Gitlab::UrlBuilder.build(deployment.user), + commit_url: Gitlab::UrlBuilder.build(deployment.commit), + commit_title: deployment.commit.title + } + end + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index af385d7d4ca..40bda3410e1 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -58,7 +58,10 @@ module Gitlab # } # # rubocop:disable Metrics/ParameterLists - def build(project, user, oldrev, newrev, ref, commits = [], message = nil, commits_count: nil, push_options: {}) + def build( + project:, user:, ref:, oldrev: nil, newrev: nil, + commits: [], commits_count: nil, message: nil, push_options: {}) + commits = Array(commits) # Total commits count @@ -113,7 +116,12 @@ module Gitlab ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" commits = project.repository.commits(project.default_branch.to_s, limit: 3) - build(project, user, commits.last&.id, commits.first&.id, ref, commits) + build(project: project, + user: user, + oldrev: commits.last&.id, + newrev: commits.first&.id, + ref: ref, + commits: commits) end def sample_data diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 270cfb89488..369c6b87fb4 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -52,6 +52,19 @@ module Gitlab end end + # Clears multiple cache keys at once. + # + # raw_keys - An Array of unique cache keys, without namespaces. + # + # It returns the number of cache keys cleared. Ex.: 42 + def clear_multiple(raw_keys) + return [] if raw_keys.empty? + + keys = raw_keys.map { |id| cache_key_for(id) } + + Redis::Cache.with { |redis| redis.del(keys) } + end + def cache_key_for(raw_key) "#{cache_key_prefix}:#{raw_key}" end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 2770469ca9f..9fc2217ad43 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -16,6 +16,7 @@ module Gitlab avatar: /\Alogo\.(png|jpg|gif)\z/, issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, + metrics_dashboard: %r{\A\.gitlab/dashboards/[^/]+\.yml\z}, xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z}, # Configuration files diff --git a/lib/gitlab/git/object_pool.rb b/lib/gitlab/git/object_pool.rb index 8eb3c28ab70..d0577d7a4ff 100644 --- a/lib/gitlab/git/object_pool.rb +++ b/lib/gitlab/git/object_pool.rb @@ -40,6 +40,10 @@ module Gitlab @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) end + def fetch + object_pool_service.fetch(source_repository) + end + private def object_pool_service diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index b46d4ba0b02..ef9b1bf5224 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -14,7 +14,7 @@ module Gitlab 'GL-HOOK-ERR:' # Messages marked as safe by user ].freeze - SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/ + SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze def initialize(message = '') super(sanitize(message)) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index be9e926728c..fc9bcbdcca2 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -118,6 +118,12 @@ module Gitlab gitaly_repository_client.exists? end + def create_repository + wrapped_gitaly_errors do + gitaly_repository_client.create_repository + end + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -231,12 +237,12 @@ module Gitlab end end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) + def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha) + prefix = archive_prefix(ref, commit.id, project_path, append_sha: append_sha, path: path) { 'ArchivePrefix' => prefix, @@ -248,13 +254,14 @@ module Gitlab # This is both the filename of the archive (missing the extension) and the # name of the top-level member of the archive under which all files go - def archive_prefix(ref, sha, project_path, append_sha:) + def archive_prefix(ref, sha, project_path, append_sha:, path:) append_sha = (ref != sha) if append_sha.nil? formatted_ref = ref.tr('/', '-') prefix_segments = [project_path, formatted_ref] prefix_segments << sha if append_sha + prefix_segments << path.tr('/', '-').gsub(%r{^/|/$}, '') if path prefix_segments.join('-') end @@ -731,18 +738,29 @@ module Gitlab end def compare_source_branch(target_branch_name, source_repository, source_branch_name, straight:) + reachable_ref = + if source_repository == self + source_branch_name + else + # If a tmp ref was created before for a separate repo comparison (forks), + # we're able to short-circuit the tmp ref re-creation: + # 1. Take the SHA from the source repo + # 2. Read that in the current "target" repo + # 3. If that SHA is still known (readable), it means GC hasn't + # cleaned it up yet, so we can use it instead re-writing the tmp ref. + source_commit_id = source_repository.commit(source_branch_name)&.sha + commit(source_commit_id)&.sha if source_commit_id + end + + return compare(target_branch_name, reachable_ref, straight: straight) if reachable_ref + tmp_ref = "refs/tmp/#{SecureRandom.hex}" return unless fetch_source_branch!(source_repository, source_branch_name, tmp_ref) - Gitlab::Git::Compare.new( - self, - target_branch_name, - tmp_ref, - straight: straight - ) + compare(target_branch_name, tmp_ref, straight: straight) ensure - delete_refs(tmp_ref) + delete_refs(tmp_ref) if tmp_ref end def write_ref(ref_path, ref, old_ref: nil) @@ -816,7 +834,8 @@ module Gitlab gitaly_repository_client.create_from_snapshot(url, auth) end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + # DEPRECATED: https://gitlab.com/gitlab-org/gitaly/issues/1628 + def rebase_deprecated(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) wrapped_gitaly_errors do gitaly_operation_client.user_rebase(user, rebase_id, branch: branch, @@ -826,6 +845,20 @@ module Gitlab end end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, &block) + wrapped_gitaly_errors do + gitaly_operation_client.rebase( + user, + rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch, + &block + ) + end + end + def rebase_in_progress?(rebase_id) wrapped_gitaly_errors do gitaly_repository_client.rebase_in_progress?(rebase_id) @@ -889,6 +922,12 @@ module Gitlab end end + def disconnect_alternates + wrapped_gitaly_errors do + gitaly_repository_client.disconnect_alternates + end + end + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository, @gl_project_path) end @@ -998,6 +1037,13 @@ module Gitlab private + def compare(base_ref, head_ref, straight:) + Gitlab::Git::Compare.new(self, + base_ref, + head_ref, + straight: straight) + end + def empty_diff_stats Gitlab::Git::DiffStatsCollection.new([]) end diff --git a/lib/gitlab/git/repository_cleaner.rb b/lib/gitlab/git/repository_cleaner.rb index 2d1d8435cf3..9dd0ddfb44b 100644 --- a/lib/gitlab/git/repository_cleaner.rb +++ b/lib/gitlab/git/repository_cleaner.rb @@ -12,9 +12,9 @@ module Gitlab @repository = repository end - def apply_bfg_object_map(io) + def apply_bfg_object_map_stream(io, &blk) wrapped_gitaly_errors do - gitaly_cleanup_client.apply_bfg_object_map(io) + gitaly_cleanup_client.apply_bfg_object_map_stream(io, &blk) end end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index f6777dfa0c3..bce4fa14fb4 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -21,6 +21,17 @@ module Gitlab nil end + # This needs to return an array of Gitlab::Git:Commit objects + # instead of Rugged::Commit objects to ensure upstream models + # operate on a consistent interface. Unlike + # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid + # doesn't attempt to decorate the result. + def rugged_batch_by_oid(repo, oids) + oids.map { |oid| rugged_find(repo, oid) } + .compact + .map { |commit| decorate(repo, commit) } + end + override :find_commit def find_commit(repo, commit_id) if Feature.enabled?(:rugged_find_commit) @@ -29,6 +40,15 @@ module Gitlab super end end + + override :batch_by_oid + def batch_by_oid(repo, oids) + if Feature.enabled?(:rugged_list_commits_by_oid) + rugged_batch_by_oid(repo, oids) + else + super + end + end end extend ::Gitlab::Utils::Override diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index c0a91f59ab9..e91b0ddcd31 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -12,7 +12,7 @@ module Gitlab module Repository extend ::Gitlab::Utils::Override - FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry).freeze + FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze def alternate_object_directories relative_object_directories.map { |d| File.join(path, d) } diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index a0dd4a24363..c1bcd8e934a 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -86,9 +86,14 @@ module Gitlab end end - def pages(limit: 0, sort: nil, direction_desc: false) + def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) wrapped_gitaly_errors do - gitaly_get_all_pages(limit: limit, sort: sort, direction_desc: direction_desc) + gitaly_list_pages( + limit: limit, + sort: sort, + direction_desc: direction_desc, + load_content: load_content + ) end end @@ -168,10 +173,17 @@ module Gitlab Gitlab::Git::WikiFile.new(wiki_file) end - def gitaly_get_all_pages(limit: 0, sort: nil, direction_desc: false) - gitaly_wiki_client.get_all_pages( - limit: limit, sort: sort, direction_desc: direction_desc - ).map do |wiki_page, version| + def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) + params = { limit: limit, sort: sort, direction_desc: direction_desc } + + gitaly_pages = + if load_content + gitaly_wiki_client.load_all_pages(params) + else + gitaly_wiki_client.list_all_pages(params) + end + + gitaly_pages.map do |wiki_page, version| Gitlab::Git::WikiPage.new(wiki_page, version) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index cb80ed64eff..4b626509008 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -85,7 +85,7 @@ module Gitlab check_push_access! end - ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages(cmd)) + success_result(cmd) end def guest_can_download_code? @@ -365,6 +365,10 @@ module Gitlab protected + def success_result(cmd) + ::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages(cmd)) + end + def changes_list @changes_list ||= Gitlab::ChangesList.new(changes == ANY ? [] : changes) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 35565b68388..05e06eec012 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,11 +26,15 @@ module Gitlab end end - PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze + # Server feature flags should use '_' to separate words. + SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE, 'delta_islands'].freeze + MUTEX = Mutex.new define_histogram :gitaly_controller_action_duration_seconds do @@ -52,9 +56,9 @@ module Gitlab end def self.interceptors - return [] unless Gitlab::Tracing.enabled? + return [] unless Labkit::Tracing.enabled? - [Gitlab::Tracing::GRPCInterceptor.instance] + [Labkit::Tracing::GRPCInterceptor.instance] end private_class_method :interceptors @@ -165,7 +169,10 @@ module Gitlab current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) - add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) + if peek_enabled? + add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) + end end def self.query_time @@ -215,7 +222,8 @@ module Gitlab feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage - metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id + metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id + metadata['gitaly-session-id'] = session_id if feature_enabled?(SERVER_FEATURE_CATFILE_CACHE) metadata.merge!(server_feature_flags) @@ -232,7 +240,9 @@ module Gitlab result end - SERVER_FEATURE_FLAGS = %w[].freeze + def self.session_id + Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid + end def self.server_feature_flags SERVER_FEATURE_FLAGS.map do |f| @@ -350,15 +360,17 @@ module Gitlab Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 end - def self.add_call_details(details) - return unless Gitlab::SafeRequestStore[:peek_enabled] + def self.peek_enabled? + Gitlab::SafeRequestStore[:peek_enabled] + end + def self.add_call_details(details) Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] Gitlab::SafeRequestStore['gitaly_call_details'] << details end def self.list_call_details - return [] unless Gitlab::SafeRequestStore[:peek_enabled] + return [] unless peek_enabled? Gitlab::SafeRequestStore['gitaly_call_details'] || [] end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 6b8e58e6199..8ccefb00d20 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -55,13 +55,13 @@ module Gitlab def get_blobs(revision_paths, limit = -1) return [] if revision_paths.empty? - revision_paths.map! do |rev, path| + request_revision_paths = revision_paths.map do |rev, path| Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) end request = Gitaly::GetBlobsRequest.new( repository: @gitaly_repo, - revision_paths: revision_paths, + revision_paths: request_revision_paths, limit: limit ) diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index 3e8d6a773ca..a56bc35f6d7 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -12,25 +12,32 @@ module Gitlab @storage = repository.storage end - def apply_bfg_object_map(io) - first_request = Gitaly::ApplyBfgObjectMapRequest.new(repository: gitaly_repo) + def apply_bfg_object_map_stream(io, &blk) + responses = GitalyClient.call( + storage, + :cleanup_service, + :apply_bfg_object_map_stream, + build_object_map_enum(io), + timeout: GitalyClient.no_timeout + ) + + responses.each(&blk) + end + + private - enum = Enumerator.new do |y| - y.yield first_request + def build_object_map_enum(io) + Enumerator.new do |y| + # First request. For simplicity, doesn't include any object map data + y << Gitaly::ApplyBfgObjectMapStreamRequest.new(repository: gitaly_repo) + # Now stream the BFG object map file to gitaly in chunks while data = io.read(RepositoryService::MAX_MSG_SIZE) - y.yield Gitaly::ApplyBfgObjectMapRequest.new(object_map: data) + y << Gitaly::ApplyBfgObjectMapStreamRequest.new(object_map: data) + break if io&.eof? end end - - GitalyClient.call( - storage, - :cleanup_service, - :apply_bfg_object_map, - enum, - timeout: GitalyClient.no_timeout - ) end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0d5debfcd01..2896b7e1ce0 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -174,7 +174,7 @@ module Gitlab response.each_with_object({}) do |gitaly_response, hsh| gitaly_response.commits.each do |commit_for_tree| - hsh[commit_for_tree.path] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) + hsh[commit_for_tree.path_bytes] = Gitlab::Git::Commit.new(@repository, commit_for_tree.commit) end end end diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index ce1fb4d68ae..d7fac26bc13 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -33,6 +33,15 @@ module Gitlab GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, request, timeout: GitalyClient.fast_timeout) end + + def fetch(repository) + request = Gitaly::FetchIntoObjectPoolRequest.new( + object_pool: object_pool, + origin: repository.gitaly_repository + ) + + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index b0f328ce3d4..e4a59ee3f9b 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -197,6 +197,7 @@ module Gitlab start_repository: start_repository) end + # DEPRECATED: https://gitlab.com/gitlab-org/gitaly/issues/1628 def user_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) request = Gitaly::UserRebaseRequest.new( repository: @gitaly_repo, @@ -225,6 +226,49 @@ module Gitlab end end + def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + request_enum = QueueEnumerator.new + rebase_sha = nil + + response_enum = GitalyClient.call( + @repository.storage, + :operation_service, + :user_rebase_confirmable, + request_enum.each, + remote_storage: remote_repository.storage + ) + + # First request + request_enum.push( + Gitaly::UserRebaseConfirmableRequest.new( + header: Gitaly::UserRebaseConfirmableRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + rebase_id: rebase_id.to_s, + branch: encode_binary(branch), + branch_sha: branch_sha, + remote_repository: remote_repository.gitaly_repository, + remote_branch: encode_binary(remote_branch) + ) + ) + ) + + perform_next_gitaly_rebase_request(response_enum) do |response| + rebase_sha = response.rebase_sha + end + + yield rebase_sha + + # Second request confirms with gitaly to finalize the rebase + request_enum.push(Gitaly::UserRebaseConfirmableRequest.new(apply: true)) + + perform_next_gitaly_rebase_request(response_enum) + + rebase_sha + ensure + request_enum.close + end + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, @@ -346,6 +390,20 @@ module Gitlab private + def perform_next_gitaly_rebase_request(response_enum) + response = response_enum.next + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.git_error.present? + raise Gitlab::Git::Repository::GitError, response.git_error + end + + yield response if block_given? + + response + end + def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 6f6698607d9..b7d509dfa48 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -239,6 +239,12 @@ module Gitlab messages end + def pack_refs + request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) + + GitalyClient.call(@storage, :ref_service, :pack_refs, request) + end + private def consume_refs_response(response) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 74aae4a8e97..68b17e86608 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -331,6 +331,14 @@ module Gitlab search_results_from_response(response) end + def disconnect_alternates + request = Gitaly::DisconnectGitAlternatesRequest.new( + repository: @gitaly_repo + ) + + GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request) + end + private def search_results_from_response(gitaly_response) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index e036cdcd800..ce9faad825c 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -87,7 +87,27 @@ module Gitlab wiki_page_from_iterator(response) end - def get_all_pages(limit: 0, sort: nil, direction_desc: false) + def list_all_pages(limit: 0, sort: nil, direction_desc: false) + sort_value = Gitaly::WikiListPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) + + params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } + params[:sort] = sort_value if sort_value + + request = Gitaly::WikiListPagesRequest.new(params) + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_list_pages, request, timeout: GitalyClient.medium_timeout) + stream.each_with_object([]) do |message, pages| + page = message.page + + next unless page + + wiki_page = GitalyClient::WikiPage.new(page.to_h) + version = new_wiki_page_version(page.version) + + pages << [wiki_page, version] + end + end + + def load_all_pages(limit: 0, sort: nil, direction_desc: false) sort_value = Gitaly::WikiGetAllPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } @@ -95,6 +115,7 @@ module Gitlab request = Gitaly::WikiGetAllPagesRequest.new(params) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) + pages = [] loop do diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 656d46b6a7d..a468f6d8821 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -53,6 +53,7 @@ module Gitlab description: description, milestone_id: milestone_finder.id_for(issue), state: issue.state, + state_id: ::Issue.available_states[issue.state], created_at: issue.created_at, updated_at: issue.updated_at } diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 1b293ddc7c7..377e873d24d 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -55,6 +55,7 @@ module Gitlab source_branch: pull_request.formatted_source_branch, target_branch: pull_request.target_branch, state: pull_request.state, + state_id: ::MergeRequest.available_states[pull_request.state], milestone_id: milestone_finder.id_for(pull_request), author_id: author_id, assignee_id: user_finder.assignee_id_for(pull_request), diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index be1334ca98a..d336b1ba797 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 070e3b2db8d..5b98ce7d5ed 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index a56ca1e39e7..04dabe423e8 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true module Gitlab - module GlRepository + class GlRepository + include Singleton + PROJECT = RepoType.new( name: :project, access_checker_class: Gitlab::GitAccess, @@ -19,7 +21,7 @@ module Gitlab }.freeze def self.types - TYPES + instance.types end def self.parse(gl_repository) @@ -39,5 +41,11 @@ module Gitlab def self.default_type PROJECT end + + def types + TYPES + end + + private_class_method :instance end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 7abe6c29a25..19915980d7f 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module GlRepository + class GlRepository class RepoType attr_reader :name, :access_checker_class, diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index e00309e7946..582c3065189 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -15,7 +15,12 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled + + if Gitlab::CurrentSettings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn + gon.sentry_environment = Gitlab.config.sentry.environment + end + gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab.revision gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') diff --git a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb index fa4c5d86d44..bbe4b24c7d4 100644 --- a/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb +++ b/lib/gitlab/grape_logging/loggers/correlation_id_logger.rb @@ -6,7 +6,7 @@ module Gitlab module Loggers class CorrelationIdLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) - { Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id } + { Labkit::Correlation::CorrelationId::LOG_KEY => Labkit::Correlation::CorrelationId.current_id } end end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb index f62813db82c..f8d0208e275 100644 --- a/lib/gitlab/graphql/authorize.rb +++ b/lib/gitlab/graphql/authorize.rb @@ -8,7 +8,7 @@ module Gitlab extend ActiveSupport::Concern def self.use(schema_definition) - schema_definition.instrument(:field, Instrumentation.new) + schema_definition.instrument(:field, Instrumentation.new, after_built_ins: true) end end end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 8deff79fc84..619ce100421 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -15,15 +15,10 @@ module Gitlab def authorized_resolve proc do |parent_typed_object, args, ctx| - resolved_obj = @old_resolve_proc.call(parent_typed_object, args, ctx) - authorizing_obj = authorize_against(parent_typed_object) - checker = build_checker(ctx[:current_user], authorizing_obj) - - if resolved_obj.respond_to?(:then) - resolved_obj.then(&checker) - else - checker.call(resolved_obj) - end + resolved_type = @old_resolve_proc.call(parent_typed_object, args, ctx) + authorizing_object = authorize_against(parent_typed_object, resolved_type) + + filter_allowed(ctx[:current_user], resolved_type, authorizing_object) end end @@ -38,7 +33,7 @@ module Gitlab type = @field.type # When the return type of @field is a collection, find the singular type - if type.get_field('edges') + if @field.connection? type = node_type_for_relay_connection(type) elsif type.list? type = node_type_for_basic_connection(type) @@ -52,43 +47,60 @@ module Gitlab Array.wrap(@field.metadata[:authorize]) end - # If it's a built-in/scalar type, authorize using its parent object. - # nil means authorize using the resolved object - def authorize_against(parent_typed_object) - parent_typed_object.object if built_in_type? && parent_typed_object.respond_to?(:object) + def authorize_against(parent_typed_object, resolved_type) + if scalar_type? + # The field is a built-in/scalar type, or a list of scalars + # authorize using the parent's object + parent_typed_object.object + elsif resolved_type.respond_to?(:object) + # The field is a type representing a single object, we'll authorize + # against the object directly + resolved_type.object + elsif @field.connection? || resolved_type.is_a?(Array) + # The field is a connection or a list of non-built-in types, we'll + # authorize each element when rendering + nil + else + # Resolved type is a single object that might not be loaded yet by + # the batchloader, we'll authorize that + resolved_type + end end - def build_checker(current_user, authorizing_obj) - lambda do |resolved_obj| - # Load the elements if they were not loaded by BatchLoader yet - resolved_obj = resolved_obj.sync if resolved_obj.respond_to?(:sync) - - check = lambda do |object| - authorizations.all? do |ability| - Ability.allowed?(current_user, ability, authorizing_obj || object) - end + def filter_allowed(current_user, resolved_type, authorizing_object) + if authorizing_object + # Authorizing fields representing scalars, or a simple field with an object + resolved_type if allowed_access?(current_user, authorizing_object) + elsif @field.connection? + # A connection with pagination, modify the visible nodes in on the + # connection type in place + resolved_type.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) } + resolved_type + elsif resolved_type.is_a? Array + # A simple list of rendered types each object being an object to authorize + resolved_type.select do |single_object_type| + allowed_access?(current_user, single_object_type.object) end + elsif resolved_type.nil? + # We're not rendering anything, for example when a record was not found + # no need to do anything + else + raise "Can't authorize #{@field}" + end + end - case resolved_obj - when Array, ActiveRecord::Relation - resolved_obj.select(&check) - else - resolved_obj if check.call(resolved_obj) - end + def allowed_access?(current_user, object) + object = object.sync if object.respond_to?(:sync) + + authorizations.all? do |ability| + Ability.allowed?(current_user, ability, object) end end # Returns the singular type for relay connections. # This will be the type class of edges.node def node_type_for_relay_connection(type) - type = type.get_field('edges').type.unwrap.get_field('node')&.type - - if type.nil? - raise Gitlab::Graphql::Errors::ConnectionDefinitionError, - 'Connection Type must conform to the Relay Cursor Connections Specification' - end - - type + type.unwrap.get_field('edges').type.unwrap.get_field('node').type end # Returns the singular type for basic connections, for example `[Types::ProjectType]` @@ -96,8 +108,8 @@ module Gitlab type.unwrap end - def built_in_type? - GraphQL::Schema::BUILT_IN_TYPES.has_value?(node_type_for_basic_connection(@field.type)) + def scalar_type? + node_type_for_basic_connection(@field.type).kind.scalar? end end end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb index 851054c0393..715963a44c1 100644 --- a/lib/gitlab/graphql/connections/keyset_connection.rb +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -22,8 +22,17 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def paged_nodes + # These are the nodes that will be loaded into memory for rendering + # So we're ok loading them into memory here as that's bound to happen + # anyway. Having them ready means we can modify the result while + # rendering the fields. + @paged_nodes ||= load_paged_nodes.to_a + end + + private + + def load_paged_nodes if first && last raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") end @@ -31,12 +40,9 @@ module Gitlab if last sliced_nodes.last(limit_value) else - sliced_nodes.limit(limit_value) + sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord end end - # rubocop: enable CodeReuse/ActiveRecord - - private def before_slice if sort_direction == :asc diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index bcbba72e017..fe74549e322 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -6,7 +6,6 @@ module Gitlab BaseError = Class.new(GraphQL::ExecutionError) ArgumentError = Class.new(BaseError) ResourceNotAvailable = Class.new(BaseError) - ConnectionDefinitionError = Class.new(BaseError) end end end diff --git a/lib/gitlab/graphql/tracing.rb b/lib/gitlab/graphql/generic_tracing.rb index 6b505e4262b..936b22d5afa 100644 --- a/lib/gitlab/graphql/tracing.rb +++ b/lib/gitlab/graphql/generic_tracing.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true +# This class is used as a hook to observe graphql runtime events. From this +# hook both gitlab metrics and opentracking measurements are generated + module Gitlab module Graphql - class Tracing < GraphQL::Tracing::PlatformTracing + class GenericTracing < GraphQL::Tracing::PlatformTracing self.platform_keys = { 'lex' => 'graphql.lex', 'parse' => 'graphql.parse', @@ -21,17 +24,30 @@ module Gitlab end def platform_trace(platform_key, key, data, &block) + tags = { platform_key: platform_key, key: key } start = Gitlab::Metrics::System.monotonic_time - yield + with_labkit_tracing(tags, &block) ensure duration = Gitlab::Metrics::System.monotonic_time - start - graphql_duration_seconds.observe({ platform_key: platform_key, key: key }, duration) + graphql_duration_seconds.observe(tags, duration) end private + def with_labkit_tracing(tags, &block) + return yield unless Labkit::Tracing.enabled? + + name = "#{tags[:platform_key]}.#{tags[:key]}" + span_tags = { + 'component' => 'web', + 'span.kind' => 'server' + }.merge(tags.stringify_keys) + + Labkit::Tracing.with_tracing(operation_name: name, tags: span_tags, &block) + end + def graphql_duration_seconds @graphql_duration_seconds ||= Gitlab::Metrics.histogram( :graphql_duration_seconds, diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 7255293b194..334642f252e 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -2,6 +2,8 @@ module Gitlab class GroupSearchResults < SearchResults + attr_reader :group + def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20) super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page) @@ -26,5 +28,9 @@ module Gitlab .where(id: groups.select('members.user_id')) end # rubocop:enable CodeReuse/ActiveRecord + + def issuable_params + super.merge(group_id: group.id) + end end end diff --git a/lib/gitlab/health_checks/metric.rb b/lib/gitlab/health_checks/metric.rb index 62a5216d159..184083de2bc 100644 --- a/lib/gitlab/health_checks/metric.rb +++ b/lib/gitlab/health_checks/metric.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/health_checks/result.rb b/lib/gitlab/health_checks/result.rb index d32a6980eb8..4586b1d94a7 100644 --- a/lib/gitlab/health_checks/result.rb +++ b/lib/gitlab/health_checks/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab::HealthChecks diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index fcf6a25ab00..acb7f225b17 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -30,10 +30,7 @@ module Gitlab def handle_response_error(response) unless response.success? - error_code = response.dig('Error', 'Code') || response.code - error_message = response.dig('Error', 'Message') || response.message - - raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}") end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index ce268793128..c6d4fda4af5 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -75,6 +75,7 @@ project_tree: - :project_badges - :ci_cd_settings - :error_tracking_setting + - :metrics_setting # Only include the following attributes for the models specified. included_attributes: diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 61a1aa6da5a..e1e70a008d9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -25,7 +25,8 @@ module Gitlab metrics: 'MergeRequest::Metrics', ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', - links: 'Releases::Link' }.freeze + links: 'Releases::Link', + metrics_setting: 'ProjectMetricsSetting' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id merged_by_id latest_closed_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb index a5a5759cc89..ab34fb03158 100644 --- a/lib/gitlab/json_logger.rb +++ b/lib/gitlab/json_logger.rb @@ -10,7 +10,7 @@ module Gitlab data = {} data[:severity] = severity data[:time] = timestamp.utc.iso8601(3) - data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id + data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id case message when String diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index 7dfd9ed4f35..ff1dadf9247 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -22,6 +22,13 @@ module Gitlab alias_method :update, :install + def uninstall(command) + namespace.ensure_exists! + + delete_pod!(command.pod_name) + kubeclient.create_pod(command.pod_resource) + end + ## # Returns Pod phase # diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb new file mode 100644 index 00000000000..876994d2678 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + class DeleteCommand + include BaseCommand + include ClientCommand + + attr_accessor :name, :files + + def initialize(name:, rbac:, files:) + @name = name + @files = files + @rbac = rbac + end + + def generate_script + super + [ + init_command, + wait_for_tiller_command, + delete_command + ].compact.join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + def rbac? + @rbac + end + + private + + def delete_command + command = ['helm', 'delete', '--purge', name] + optional_tls_flags + + command.shelljoin + end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb new file mode 100644 index 00000000000..94aabd0466c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/base_service.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +module Gitlab + module Metrics + module Dashboard + class BaseService < ::BaseService + DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError + + def get_dashboard + return error("#{dashboard_path} could not be found.", :not_found) unless path_available? + + success(dashboard: process_dashboard) + rescue DASHBOARD_LAYOUT_ERROR => e + error(e.message, :unprocessable_entity) + end + + # Summary of all known dashboards for the service. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def all_dashboard_paths(_project) + raise NotImplementedError + end + + private + + # Returns a new dashboard Hash, supplemented with DB info + def process_dashboard + Gitlab::Metrics::Dashboard::Processor + .new(project, params[:environment], raw_dashboard) + .process(insert_project_metrics: insert_project_metrics?) + end + + # @return [String] Relative filepath of the dashboard yml + def dashboard_path + params[:dashboard_path] + end + + # Returns an un-processed dashboard from the cache. + def raw_dashboard + Rails.cache.fetch(cache_key) { get_raw_dashboard } + end + + # @return [Hash] an unmodified dashboard + def get_raw_dashboard + raise NotImplementedError + end + + # @return [String] + def cache_key + raise NotImplementedError + end + + # Determines whether custom metrics should be included + # in the processed output. + def insert_project_metrics? + false + end + + # Checks if dashboard path exists or should be rejected + # as a result of file-changes to the project repository. + # @return [Boolean] + def path_available? + available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project) + + available_paths.any? do |path_params| + path_params[:path] == dashboard_path + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb new file mode 100644 index 00000000000..4a41590f000 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# Returns DB-supplmented dashboard info for determining +# the layout of UI. Intended entry-point for the Metrics::Dashboard +# module. +module Gitlab + module Metrics + module Dashboard + class Finder + class << self + # Returns a formatted dashboard packed with DB info. + # @return [Hash] + def find(project, user, environment, dashboard_path = nil) + service = system_dashboard?(dashboard_path) ? system_service : project_service + + service + .new(project, user, environment: environment, dashboard_path: dashboard_path) + .get_dashboard + end + + # Summary of all known dashboards. + # @return [Array<Hash>] ex) [{ path: String, default: Boolean }] + def find_all_paths(project) + project.repository.metrics_dashboard_paths + end + + # Summary of all known dashboards. Used to populate repo cache. + # Prefer #find_all_paths. + def find_all_paths_from_source(project) + system_service.all_dashboard_paths(project) + .+ project_service.all_dashboard_paths(project) + end + + private + + def system_service + Gitlab::Metrics::Dashboard::SystemDashboardService + end + + def project_service + Gitlab::Metrics::Dashboard::ProjectDashboardService + end + + def system_dashboard?(filepath) + !filepath || system_service.system_dashboard?(filepath) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb new file mode 100644 index 00000000000..dd986020693 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/processor.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + # Responsible for processesing a dashboard hash, inserting + # relevant DB records & sorting for proper rendering in + # the UI. These includes shared metric info, custom metrics + # info, and alerts (only in EE). + class Processor + SYSTEM_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::ProjectMetricsInserter, + Stages::Sorter + ].freeze + + PROJECT_SEQUENCE = [ + Stages::CommonMetricsInserter, + Stages::Sorter + ].freeze + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Returns a new dashboard hash with the results of + # running transforms on the dashboard. + def process(insert_project_metrics:) + @dashboard.deep_symbolize_keys.tap do |dashboard| + sequence(insert_project_metrics).each do |stage| + stage.new(@project, @environment, dashboard).transform! + end + end + end + + private + + def sequence(insert_project_metrics) + insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb new file mode 100644 index 00000000000..fdffd067c93 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Searches a projects repository for a metrics dashboard and formats the output. +# Expects any custom dashboards will be located in `.gitlab/dashboards` +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService + DASHBOARD_ROOT = ".gitlab/dashboards" + + class << self + def all_dashboard_paths(project) + file_finder(project) + .list_files_for(DASHBOARD_ROOT) + .map do |filepath| + Rails.cache.delete(cache_key(project.id, filepath)) + + { path: filepath, default: false } + end + end + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml') + end + + def cache_key(id, dashboard_path) + "project_#{id}_metrics_dashboard_#{dashboard_path}" + end + end + + private + + # Searches the project repo for a custom-defined dashboard. + def get_raw_dashboard + yml = self.class.file_finder(project).read(dashboard_path) + + YAML.safe_load(yml) + end + + def cache_key + self.class.cache_key(project.id, dashboard_path) + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb new file mode 100644 index 00000000000..a6d1f974556 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class BaseStage + DashboardLayoutError = Class.new(StandardError) + + DEFAULT_PANEL_TYPE = 'area-chart' + + attr_reader :project, :environment, :dashboard + + def initialize(project, environment, dashboard) + @project = project + @environment = environment + @dashboard = dashboard + end + + # Entry-point to the stage + def transform! + raise NotImplementedError + end + + protected + + def missing_panel_groups! + raise DashboardLayoutError.new('Top-level key :panel_groups must be an array') + end + + def missing_panels! + raise DashboardLayoutError.new('Each "panel_group" must define an array :panels') + end + + def missing_metrics! + raise DashboardLayoutError.new('Each "panel" must define an array :metrics') + end + + def for_metrics + missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) + + dashboard[:panel_groups].each do |panel_group| + missing_panels! unless panel_group[:panels].is_a?(Array) + + panel_group[:panels].each do |panel| + missing_metrics! unless panel[:metrics].is_a?(Array) + + panel[:metrics].each do |metric| + yield metric + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb new file mode 100644 index 00000000000..188912bedb4 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class CommonMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, + # includes the record's id in the dashboard config. + def transform! + common_metrics = ::PrometheusMetric.common + + for_metrics do |metric| + metric_record = common_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb new file mode 100644 index 00000000000..221610a14d1 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/project_metrics_inserter.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class ProjectMetricsInserter < BaseStage + # Inserts project-specific metrics into the dashboard + # config. If there are no project-specific metrics, + # this will have no effect. + def transform! + project.prometheus_metrics.each do |project_metric| + group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) + panel = find_or_create_panel(group[:panels], project_metric) + find_or_create_metric(panel[:metrics], project_metric) + end + end + + private + + # Looks for a panel_group corresponding to the + # provided metric object. If unavailable, inserts one. + # @param panel_groups [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel_group(panel_groups, metric) + panel_group = find_panel_group(panel_groups, metric) + return panel_group if panel_group + + panel_group = new_panel_group(metric) + panel_groups << panel_group + + panel_group + end + + # Looks for a panel corresponding to the provided + # metric object. If unavailable, inserts one. + # @param panels [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_panel(panels, metric) + panel = find_panel(panels, metric) + return panel if panel + + panel = new_panel(metric) + panels << panel + + panel + end + + # Looks for a metric corresponding to the provided + # metric object. If unavailable, inserts one. + # @param metrics [Array<Hash>] + # @param metric [PrometheusMetric] + def find_or_create_metric(metrics, metric) + target_metric = find_metric(metrics, metric) + return target_metric if target_metric + + target_metric = new_metric(metric) + metrics << target_metric + + target_metric + end + + def find_panel_group(panel_groups, metric) + return unless panel_groups + + panel_groups.find { |group| group[:group] == metric.group_title } + end + + def find_panel(panels, metric) + return unless panels + + panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] + panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } + end + + def find_metric(metrics, metric) + return unless metrics + + metrics.find { |m| m[:id] == metric.identifier } + end + + def new_panel_group(metric) + { + group: metric.group_title, + priority: metric.priority, + panels: [] + } + end + + def new_panel(metric) + { + type: DEFAULT_PANEL_TYPE, + title: metric.title, + y_label: metric.y_label, + metrics: [] + } + end + + def new_metric(metric) + metric.queries.first.merge(metric_id: metric.id) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb new file mode 100644 index 00000000000..ba5aa78059c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class Sorter < BaseStage + def transform! + missing_panel_groups! unless dashboard[:panel_groups].is_a? Array + + sort_groups! + sort_panels! + end + + private + + # Sorts the groups in the dashboard by the :priority key + def sort_groups! + dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i } + end + + # Sorts the panels in the dashboard by the :weight key + def sort_panels! + dashboard[:panel_groups].each do |group| + missing_panels! unless group[:panels].is_a? Array + + group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i } + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb new file mode 100644 index 00000000000..67509ed4230 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# Fetches the system metrics dashboard and formats the output. +# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards. +module Gitlab + module Metrics + module Dashboard + class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService + SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml' + + class << self + def all_dashboard_paths(_project) + [{ + path: SYSTEM_DASHBOARD_PATH, + default: true + }] + end + + def system_dashboard?(filepath) + filepath == SYSTEM_DASHBOARD_PATH + end + end + + private + + def dashboard_path + SYSTEM_DASHBOARD_PATH + end + + # Returns the base metrics shipped with every GitLab service. + def get_raw_dashboard + yml = File.read(Rails.root.join(dashboard_path)) + + YAML.safe_load(yml) + end + + def cache_key + "metrics_dashboard_#{dashboard_path}" + end + + def insert_project_metrics? + true + end + end + end + end +end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 9e4d70a71ff..30f181542be 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics # Class for storing details of a single metric (label, value, etc). class Metric - JITTER_RANGE = 0.000001..0.001 + JITTER_RANGE = (0.000001..0.001).freeze attr_reader :series, :values, :tags, :type diff --git a/lib/gitlab/middleware/correlation_id.rb b/lib/gitlab/middleware/correlation_id.rb index 80dddc41c12..fffd5da827f 100644 --- a/lib/gitlab/middleware/correlation_id.rb +++ b/lib/gitlab/middleware/correlation_id.rb @@ -12,7 +12,7 @@ module Gitlab end def call(env) - ::Gitlab::CorrelationId.use_id(correlation_id(env)) do + ::Labkit::Correlation::CorrelationId.use_id(correlation_id(env)) do @app.call(env) end end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 83c52a6c6e0..8e17073abab 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -3,7 +3,7 @@ module Gitlab module Middleware class ReadOnly - API_VERSIONS = (3..4) + API_VERSIONS = (3..4).freeze def self.internal_routes @internal_routes ||= diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb index 849cf8f759b..0719fb2e8c6 100644 --- a/lib/gitlab/middleware/release_env.rb +++ b/lib/gitlab/middleware/release_env.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/namespaced_session_store.rb b/lib/gitlab/namespaced_session_store.rb new file mode 100644 index 00000000000..34520078bfb --- /dev/null +++ b/lib/gitlab/namespaced_session_store.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + class NamespacedSessionStore + delegate :[], :[]=, to: :store + + def initialize(key) + @key = key + end + + def initiated? + !Session.current.nil? + end + + def store + return unless Session.current + + Session.current[@key] ||= {} + Session.current[@key] + end + end +end diff --git a/lib/gitlab/performance_bar/peek_query_tracker.rb b/lib/gitlab/performance_bar/peek_query_tracker.rb index ac392432427..16c16aa0265 100644 --- a/lib/gitlab/performance_bar/peek_query_tracker.rb +++ b/lib/gitlab/performance_bar/peek_query_tracker.rb @@ -26,11 +26,7 @@ module Gitlab subscribe('sql.active_record') do |_, start, finish, _, data| if Gitlab::SafeRequestStore.store[:peek_enabled] - # data[:cached] is only available starting from Rails 5.1.0 - # https://github.com/rails/rails/blob/v5.1.0/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L113 - # Before that, data[:name] was set to 'CACHE' - # https://github.com/rails/rails/blob/v4.2.9/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb#L80 - unless data.fetch(:cached, data[:name] == 'CACHE') + unless data[:cached] track_query(data[:sql].strip, data[:binds], start, finish) end end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 93a9fcf1591..890228e5e78 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -16,7 +16,11 @@ module Gitlab ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ lib/gitlab/request_profiler/ + lib/gitlab/query_limiting/ + lib/gitlab/tracing/ lib/gitlab/profiler.rb + lib/gitlab/correlation_id.rb + lib/gitlab/webpack/dev_server_middleware.rb ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) @@ -69,7 +73,7 @@ module Gitlab result = with_custom_logger(logger) do with_user(user) do - RubyProf.profile { app.public_send(verb, url, post_data, headers) } # rubocop:disable GitlabSecurity/PublicSend + RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 58f06b6708c..78337518988 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -145,5 +145,9 @@ module Gitlab def repository_wiki_ref @repository_wiki_ref ||= repository_ref || project.wiki.default_branch end + + def issuable_params + super.merge(project_id: project.id) + end end end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 1cc85d4b4a6..dca09aef47d 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -4,9 +4,13 @@ module Gitlab module Prometheus module QueryVariables def self.call(environment) + deployment_platform = environment.deployment_platform + namespace = deployment_platform&.namespace_for(environment.project) || + deployment_platform&.actual_namespace || '' + { ci_environment_slug: environment.slug, - kube_namespace: environment.deployment_platform&.actual_namespace || '', + kube_namespace: namespace, environment_filter: %{container_name!="POD",environment="#{environment.slug}"} } end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 810aba436cc..3137676ba4b 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -15,7 +15,7 @@ module Gitlab mr: :merge_request }).freeze - OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/ + OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze attr_reader :options diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb index 62c0fbb5afd..1018910e8e9 100644 --- a/lib/gitlab/quick_actions/commit_actions.rb +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -8,10 +8,13 @@ module Gitlab included do # Commit only quick actions definitions - desc 'Tag this commit.' + desc _('Tag this commit.') explanation do |tag_name, message| - with_message = %{ with "#{message}"} if message.present? - "Tags this commit to #{tag_name}#{with_message}." + if message.present? + _("Tags this commit to %{tag_name} with \"%{message}\".") % { tag_name: tag_name, message: message } + else + _("Tags this commit to %{tag_name}.") % { tag_name: tag_name } + end end params 'v1.2.3 <message>' parse_params do |tag_name_and_message| diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index ad2e15d19fa..572c55efcc2 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -43,9 +43,9 @@ module Gitlab @updates[:state_event] = 'reopen' end - desc 'Change title' + desc _('Change title') explanation do |title_param| - "Changes the title to \"#{title_param}\"." + _("Changes the title to \"%{title_param}\".") % { title_param: title_param } end params '<New title>' types Issuable @@ -57,7 +57,7 @@ module Gitlab @updates[:title] = title_param end - desc 'Add label(s)' + desc _('Add label(s)') explanation do |labels_param| labels = find_label_references(labels_param) @@ -81,13 +81,13 @@ module Gitlab end end - desc 'Remove all or specific label(s)' + desc _('Remove all or specific label(s)') explanation do |labels_param = nil| if labels_param.present? labels = find_label_references(labels_param) "Removes #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? else - 'Removes all labels.' + _('Removes all labels.') end end params '~label1 ~"label 2"' @@ -112,7 +112,7 @@ module Gitlab end end - desc 'Replace all label(s)' + desc _('Replace all label(s)') explanation do |labels_param| labels = find_label_references(labels_param) "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? @@ -135,8 +135,8 @@ module Gitlab end end - desc 'Add a todo' - explanation 'Adds a todo.' + desc _('Add a todo') + explanation _('Adds a todo.') types Issuable condition do quick_action_target.persisted? && @@ -146,8 +146,8 @@ module Gitlab @updates[:todo_event] = 'add' end - desc 'Mark todo as done' - explanation 'Marks todo as done.' + desc _('Mark todo as done') + explanation _('Marks todo as done.') types Issuable condition do quick_action_target.persisted? && @@ -157,7 +157,7 @@ module Gitlab @updates[:todo_event] = 'done' end - desc 'Subscribe' + desc _('Subscribe') explanation do "Subscribes to this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." end @@ -170,7 +170,7 @@ module Gitlab @updates[:subscription_event] = 'subscribe' end - desc 'Unsubscribe' + desc _('Unsubscribe') explanation do "Unsubscribes from this #{quick_action_target.to_ability_name.humanize(capitalize: false)}." end @@ -183,9 +183,9 @@ module Gitlab @updates[:subscription_event] = 'unsubscribe' end - desc 'Toggle emoji award' + desc _('Toggle emoji award') explanation do |name| - "Toggles :#{name}: emoji award." if name + _("Toggles :%{name}: emoji award.") % { name: name } if name end params ':emoji:' types Issuable @@ -202,14 +202,14 @@ module Gitlab end end - desc "Append the comment with #{SHRUG}" + desc _("Append the comment with %{shrug}") % { shrug: SHRUG } params '<Comment>' types Issuable substitution :shrug do |comment| "#{comment} #{SHRUG}" end - desc "Append the comment with #{TABLEFLIP}" + desc _("Append the comment with %{TABLEFLIP}") % { tableflip: TABLEFLIP } params '<Comment>' types Issuable substitution :tableflip do |comment| diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 1f08e8740a2..85e62f950c8 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -8,9 +8,9 @@ module Gitlab included do # Issue only quick actions definition - desc 'Set due date' + desc _('Set due date') explanation do |due_date| - "Sets the due date to #{due_date.to_s(:medium)}." if due_date + _("Sets the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date end params '<in 2 days | this Friday | December 31st>' types Issue @@ -25,8 +25,8 @@ module Gitlab @updates[:due_date] = due_date if due_date end - desc 'Remove due date' - explanation 'Removes the due date.' + desc _('Remove due date') + explanation _('Removes the due date.') types Issue condition do quick_action_target.persisted? && @@ -38,10 +38,10 @@ module Gitlab @updates[:due_date] = nil end - desc 'Move issue from one column of the board to another' + desc _('Move issue from one column of the board to another') explanation do |target_list_name| label = find_label_references(target_list_name).first - "Moves issue to #{label} column in the board." if label + _("Moves issue to %{label} column in the board.") % { label: label } if label end params '~"Target column"' types Issue @@ -66,9 +66,9 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - desc 'Mark this issue as a duplicate of another issue' + desc _('Mark this issue as a duplicate of another issue') explanation do |duplicate_reference| - "Marks this issue as a duplicate of #{duplicate_reference}." + _("Marks this issue as a duplicate of %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } end params '#issue' types Issue @@ -84,9 +84,9 @@ module Gitlab end end - desc 'Move this issue to another project.' + desc _('Move this issue to another project.') explanation do |path_to_project| - "Moves this issue to #{path_to_project}." + _("Moves this issue to %{path_to_project}.") % { path_to_project: path_to_project } end params 'path/to/project' types Issue @@ -102,9 +102,9 @@ module Gitlab end end - desc 'Make issue confidential.' + desc _('Make issue confidential.') explanation do - 'Makes this issue confidential' + _('Makes this issue confidential') end types Issue condition do @@ -114,10 +114,13 @@ module Gitlab @updates[:confidential] = true end - desc 'Create a merge request.' + desc _('Create a merge request.') explanation do |branch_name = nil| - branch_text = branch_name ? "branch '#{branch_name}'" : 'a branch' - "Creates #{branch_text} and a merge request to resolve this issue" + if branch_name + _("Creates branch '%{branch_name}' and a merge request to resolve this issue") % { branch_name: branch_name } + else + "Creates a branch and a merge request to resolve this issue" + end end params "<branch name>" types Issue diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 08872eda410..1cd158db2b0 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -8,7 +8,7 @@ module Gitlab included do # Issue, MergeRequest: quick actions definitions - desc 'Assign' + desc _('Assign') # rubocop: disable CodeReuse/ActiveRecord explanation do |users| users = quick_action_target.allows_multiple_assignees? ? users : users.take(1) @@ -38,9 +38,9 @@ module Gitlab desc do if quick_action_target.allows_multiple_assignees? - 'Remove all or specific assignee(s)' + _('Remove all or specific assignee(s)') else - 'Remove assignee' + _('Remove assignee') end end explanation do |users = nil| @@ -70,9 +70,9 @@ module Gitlab end end - desc 'Set milestone' + desc _('Set milestone') explanation do |milestone| - "Sets the milestone to #{milestone.to_reference}." if milestone + _("Sets the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone end params '%"milestone"' types Issue, MergeRequest @@ -88,9 +88,9 @@ module Gitlab @updates[:milestone_id] = milestone.id if milestone end - desc 'Remove milestone' + desc _('Remove milestone') explanation do - "Removes #{quick_action_target.milestone.to_reference(format: :name)} milestone." + _("Removes %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } end types Issue, MergeRequest condition do @@ -102,9 +102,9 @@ module Gitlab @updates[:milestone_id] = nil end - desc 'Copy labels and milestone from other issue or merge request' + desc _('Copy labels and milestone from other issue or merge request') explanation do |source_issuable| - "Copy labels and milestone from #{source_issuable.to_reference}." + _("Copy labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end params '#issue | !merge_request' types Issue, MergeRequest @@ -122,11 +122,11 @@ module Gitlab end end - desc 'Set time estimate' + desc _('Set time estimate') explanation do |time_estimate| time_estimate = Gitlab::TimeTrackingFormatter.output(time_estimate) - "Sets time estimate to #{time_estimate}." if time_estimate + _("Sets time estimate to %{time_estimate}.") % { time_estimate: time_estimate } if time_estimate end params '<1w 3d 2h 14m>' types Issue, MergeRequest @@ -142,18 +142,18 @@ module Gitlab end end - desc 'Add or subtract spent time' + desc _('Add or subtract spent time') explanation do |time_spent, time_spent_date| if time_spent if time_spent > 0 - verb = 'Adds' + verb = _('Adds') value = time_spent else - verb = 'Subtracts' + verb = _('Subtracts') value = -time_spent end - "#{verb} #{Gitlab::TimeTrackingFormatter.output(value)} spent time." + _("%{verb} %{time_spent_value} spent time.") % { verb: verb, time_spent_value: Gitlab::TimeTrackingFormatter.output(value) } end end params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>' @@ -174,8 +174,8 @@ module Gitlab end end - desc 'Remove time estimate' - explanation 'Removes time estimate.' + desc _('Remove time estimate') + explanation _('Removes time estimate.') types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -185,8 +185,8 @@ module Gitlab @updates[:time_estimate] = 0 end - desc 'Remove spent time' - explanation 'Removes spent time.' + desc _('Remove spent time') + explanation _('Removes spent time.') condition do quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) @@ -196,8 +196,8 @@ module Gitlab @updates[:spend_time] = { duration: :reset, user_id: current_user.id } end - desc "Lock the discussion" - explanation "Locks the discussion" + desc _("Lock the discussion") + explanation _("Locks the discussion") types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -208,8 +208,8 @@ module Gitlab @updates[:discussion_locked] = true end - desc "Unlock the discussion" - explanation "Unlocks the discussion" + desc _("Unlock the discussion") + explanation _("Unlocks the discussion") types Issue, MergeRequest condition do quick_action_target.persisted? && diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index f5176376a60..4a62e83e8e9 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -11,7 +11,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}.freeze def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index 0d4e6be2129..98f78c5e74b 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -9,7 +9,7 @@ module Gitlab class Scrubber < Loofah::Scrubber # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes - DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u + DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u.freeze def scrub(node) unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 8988b9ad7be..4a097a00101 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -2,6 +2,8 @@ module Gitlab class SearchResults + COUNT_LIMIT = 1001 + attr_reader :current_user, :query, :per_page # Limit search results by passed projects @@ -25,29 +27,26 @@ module Gitlab def objects(scope, page = nil, without_count = true) collection = case scope when 'projects' - projects.page(page).per(per_page) + projects when 'issues' - issues.page(page).per(per_page) + issues when 'merge_requests' - merge_requests.page(page).per(per_page) + merge_requests when 'milestones' - milestones.page(page).per(per_page) + milestones when 'users' - users.page(page).per(per_page) + users else - Kaminari.paginate_array([]).page(page).per(per_page) - end + Kaminari.paginate_array([]) + end.page(page).per(per_page) without_count ? collection.without_count : collection end - # rubocop: disable CodeReuse/ActiveRecord def limited_projects_count - @limited_projects_count ||= projects.limit(count_limit).count + @limited_projects_count ||= limited_count(projects) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_issues_count return @limited_issues_count if @limited_issues_count @@ -56,35 +55,28 @@ module Gitlab # and confidential issues user has access to, is too complex. # It's faster to try to fetch all public issues first, then only # if necessary try to fetch all issues. - sum = issues(public_only: true).limit(count_limit).count - @limited_issues_count = sum < count_limit ? issues.limit(count_limit).count : sum + sum = limited_count(issues(public_only: true)) + @limited_issues_count = sum < count_limit ? limited_count(issues) : sum end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_merge_requests_count - @limited_merge_requests_count ||= merge_requests.limit(count_limit).count + @limited_merge_requests_count ||= limited_count(merge_requests) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def limited_milestones_count - @limited_milestones_count ||= milestones.limit(count_limit).count + @limited_milestones_count ||= limited_count(milestones) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop:disable CodeReuse/ActiveRecord def limited_users_count - @limited_users_count ||= users.limit(count_limit).count + @limited_users_count ||= limited_count(users) end - # rubocop:enable CodeReuse/ActiveRecord def single_commit_result? false end def count_limit - 1001 + COUNT_LIMIT end def users @@ -99,49 +91,33 @@ module Gitlab limit_projects.search(query) end - # rubocop: disable CodeReuse/ActiveRecord def issues(finder_params = {}) - issues = IssuesFinder.new(current_user, finder_params).execute + issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute + unless default_project_filter - issues = issues.where(project_id: project_ids_relation) + issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - issues = - if query =~ /#(\d+)\z/ - issues.where(iid: $1) - else - issues.full_search(query) - end - - issues.reorder('updated_at DESC') + issues end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def milestones milestones = Milestone.where(project_id: project_ids_relation) milestones = milestones.search(query) - milestones.reorder('updated_at DESC') + milestones.reorder('milestones.updated_at DESC') end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def merge_requests - merge_requests = MergeRequestsFinder.new(current_user).execute + merge_requests = MergeRequestsFinder.new(current_user, issuable_params).execute + unless default_project_filter merge_requests = merge_requests.in_projects(project_ids_relation) end - merge_requests = - if query =~ /[#!](\d+)\z/ - merge_requests.where(iid: $1) - else - merge_requests.full_search(query) - end - - merge_requests.reorder('updated_at DESC') + merge_requests end - # rubocop: enable CodeReuse/ActiveRecord def default_scope 'projects' @@ -152,5 +128,23 @@ module Gitlab limit_projects.select(:id).reorder(nil) end # rubocop: enable CodeReuse/ActiveRecord + + def issuable_params + {}.tap do |params| + params[:sort] = 'updated_desc' + + if query =~ /#(\d+)\z/ + params[:iids] = $1 + else + params[:search] = query + end + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def limited_count(relation) + relation.reorder(nil).limit(count_limit).size + end + # rubocop: enable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 956c16117f5..356e6445e0e 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -45,7 +45,7 @@ module Gitlab context # Make sure we've set everything we know in the context tags = { - Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id } Raven.capture_exception(exception, tags: tags, extra: extra) diff --git a/lib/gitlab/session.rb b/lib/gitlab/session.rb new file mode 100644 index 00000000000..7487ba04a6d --- /dev/null +++ b/lib/gitlab/session.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + class Session + STORE_KEY = :session_storage + + class << self + def with_session(session) + old = self.current + self.current = session + yield + ensure + self.current = old + end + + def current + Thread.current[STORE_KEY] + end + + protected + + def current=(value) + Thread.current[STORE_KEY] = value + end + end + end +end diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 747cb0f9142..f7b08d58e49 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -4,9 +4,9 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = %r{text/html|application/json}i + CONTENT_TYPES = %r{text/html|application/json}i.freeze - IGNORE_PATHS = %r{^/sherlock} + IGNORE_PATHS = %r{^/sherlock}.freeze def initialize(app) @app = app diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 11561eec32a..159ce27e702 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -15,7 +15,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix.freeze # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index fb303e3fb0c..c102fa14cfc 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -7,7 +7,7 @@ module Gitlab module SidekiqConfig QUEUE_CONFIG_PATHS = %w[app/workers/all_queues.yml ee/app/workers/all_queues.yml].freeze - # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. def self.worker_queues(rails_path = Rails.root.to_s) @worker_queues ||= {} @@ -19,7 +19,7 @@ module Gitlab end end - # This method is called by `bin/sidekiq-cluster` in EE, which runs outside + # This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside # of bundler/Rails context, so we cannot use any gem or Rails methods. def self.expand_queues(queues, all_queues = self.worker_queues) return [] if queues.empty? diff --git a/lib/gitlab/sidekiq_middleware/correlation_injector.rb b/lib/gitlab/sidekiq_middleware/correlation_injector.rb index b807b3a03ed..1539fd706ab 100644 --- a/lib/gitlab/sidekiq_middleware/correlation_injector.rb +++ b/lib/gitlab/sidekiq_middleware/correlation_injector.rb @@ -4,8 +4,8 @@ module Gitlab module SidekiqMiddleware class CorrelationInjector def call(worker_class, job, queue, redis_pool) - job[Gitlab::CorrelationId::LOG_KEY] ||= - Gitlab::CorrelationId.current_or_new_id + job[Labkit::Correlation::CorrelationId::LOG_KEY] ||= + Labkit::Correlation::CorrelationId.current_or_new_id yield end diff --git a/lib/gitlab/sidekiq_middleware/correlation_logger.rb b/lib/gitlab/sidekiq_middleware/correlation_logger.rb index cb8ff4a6284..cffc4483573 100644 --- a/lib/gitlab/sidekiq_middleware/correlation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/correlation_logger.rb @@ -4,9 +4,9 @@ module Gitlab module SidekiqMiddleware class CorrelationLogger def call(worker, job, queue) - correlation_id = job[Gitlab::CorrelationId::LOG_KEY] + correlation_id = job[Labkit::Correlation::CorrelationId::LOG_KEY] - Gitlab::CorrelationId.use_id(correlation_id) do + Labkit::Correlation::CorrelationId.use_id(correlation_id) do yield end end diff --git a/lib/gitlab/slash_commands/result.rb b/lib/gitlab/slash_commands/result.rb index 607c9c8dec1..a66a2e0726b 100644 --- a/lib/gitlab/slash_commands/result.rb +++ b/lib/gitlab/slash_commands/result.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index b698391c8bd..fd108b4c124 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/ + REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do def fuzzy_search(query, columns) diff --git a/lib/gitlab/tracing/common.rb b/lib/gitlab/tracing/common.rb deleted file mode 100644 index 3a08ede8138..00000000000 --- a/lib/gitlab/tracing/common.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -require 'opentracing' - -module Gitlab - module Tracing - module Common - def tracer - OpenTracing.global_tracer - end - - # Convience method for running a block with a span - def in_tracing_span(operation_name:, tags:, child_of: nil) - scope = tracer.start_active_span( - operation_name, - child_of: child_of, - tags: tags - ) - span = scope.span - - # Add correlation details to the span if we have them - correlation_id = Gitlab::CorrelationId.current_id - if correlation_id - span.set_tag('correlation_id', correlation_id) - end - - begin - yield span - rescue => e - log_exception_on_span(span, e) - raise e - ensure - scope.close - end - end - - def postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil) - span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of) - - log_exception_on_span(span, exception) if exception - - span.finish(end_time: end_time) - end - - def log_exception_on_span(span, exception) - span.set_tag('error', true) - span.log_kv(kv_tags_for_exception(exception)) - end - - def kv_tags_for_exception(exception) - case exception - when Exception - { - 'event': 'error', - 'error.kind': exception.class.to_s, - 'message': Gitlab::UrlSanitizer.sanitize(exception.message), - 'stack': exception.backtrace&.join("\n") - } - else - { - 'event': 'error', - 'error.kind': exception.class.to_s, - 'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s) - } - end - end - end - end -end diff --git a/lib/gitlab/tracing/factory.rb b/lib/gitlab/tracing/factory.rb deleted file mode 100644 index fc714164353..00000000000 --- a/lib/gitlab/tracing/factory.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require "cgi" - -module Gitlab - module Tracing - class Factory - OPENTRACING_SCHEME = "opentracing" - - def self.create_tracer(service_name, connection_string) - return unless connection_string.present? - - begin - opentracing_details = parse_connection_string(connection_string) - driver_name = opentracing_details[:driver_name] - - case driver_name - when "jaeger" - JaegerFactory.create_tracer(service_name, opentracing_details[:options]) - else - raise "Unknown driver: #{driver_name}" - end - rescue => e - # Can't create the tracer? Warn and continue sans tracer - warn "Unable to instantiate tracer: #{e}" - nil - end - end - - def self.parse_connection_string(connection_string) - parsed = URI.parse(connection_string) - - unless valid_uri?(parsed) - raise "Invalid tracing connection string" - end - - { - driver_name: parsed.host, - options: parse_query(parsed.query) - } - end - private_class_method :parse_connection_string - - def self.parse_query(query) - return {} unless query - - CGI.parse(query).symbolize_keys.transform_values(&:first) - end - private_class_method :parse_query - - def self.valid_uri?(uri) - return false unless uri - - uri.scheme == OPENTRACING_SCHEME && - uri.host.to_s =~ /^[a-z0-9_]+$/ && - uri.path.empty? - end - private_class_method :valid_uri? - end - end -end diff --git a/lib/gitlab/tracing/grpc_interceptor.rb b/lib/gitlab/tracing/grpc_interceptor.rb deleted file mode 100644 index 6c2aab73125..00000000000 --- a/lib/gitlab/tracing/grpc_interceptor.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'opentracing' -require 'grpc' - -module Gitlab - module Tracing - class GRPCInterceptor < GRPC::ClientInterceptor - include Common - include Singleton - - def request_response(request:, call:, method:, metadata:) - wrap_with_tracing(method, 'unary', metadata) do - yield - end - end - - def client_streamer(requests:, call:, method:, metadata:) - wrap_with_tracing(method, 'client_stream', metadata) do - yield - end - end - - def server_streamer(request:, call:, method:, metadata:) - wrap_with_tracing(method, 'server_stream', metadata) do - yield - end - end - - def bidi_streamer(requests:, call:, method:, metadata:) - wrap_with_tracing(method, 'bidi_stream', metadata) do - yield - end - end - - private - - def wrap_with_tracing(method, grpc_type, metadata) - tags = { - 'component' => 'grpc', - 'span.kind' => 'client', - 'grpc.method' => method, - 'grpc.type' => grpc_type - } - - in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span| - OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata) - - yield - end - end - end - end -end diff --git a/lib/gitlab/tracing/jaeger_factory.rb b/lib/gitlab/tracing/jaeger_factory.rb deleted file mode 100644 index 93520d5667b..00000000000 --- a/lib/gitlab/tracing/jaeger_factory.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -require 'jaeger/client' - -module Gitlab - module Tracing - class JaegerFactory - # When the probabilistic sampler is used, by default 0.1% of requests will be traced - DEFAULT_PROBABILISTIC_RATE = 0.001 - - # The default port for the Jaeger agent UDP listener - DEFAULT_UDP_PORT = 6831 - - # Reduce this from default of 10 seconds as the Ruby jaeger - # client doesn't have overflow control, leading to very large - # messages which fail to send over UDP (max packet = 64k) - # Flush more often, with smaller packets - FLUSH_INTERVAL = 5 - - def self.create_tracer(service_name, options) - kwargs = { - service_name: service_name, - sampler: get_sampler(options[:sampler], options[:sampler_param]), - reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint]) - }.compact - - extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord - if extra_params.present? - message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}" - - if options[:strict_parsing] - raise message - else - warn message - end - end - - Jaeger::Client.build(kwargs) - end - - def self.get_sampler(sampler_type, sampler_param) - case sampler_type - when "probabilistic" - sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE - Jaeger::Samplers::Probabilistic.new(rate: sampler_rate) - when "const" - const_value = sampler_param == "1" - Jaeger::Samplers::Const.new(const_value) - else - nil - end - end - private_class_method :get_sampler - - def self.get_reporter(service_name, http_endpoint, udp_endpoint) - encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name) - - if http_endpoint.present? - sender = get_http_sender(encoder, http_endpoint) - elsif udp_endpoint.present? - sender = get_udp_sender(encoder, udp_endpoint) - else - return - end - - Jaeger::Reporters::RemoteReporter.new( - sender: sender, - flush_interval: FLUSH_INTERVAL - ) - end - private_class_method :get_reporter - - def self.get_http_sender(encoder, address) - Jaeger::HttpSender.new( - url: address, - encoder: encoder, - logger: Logger.new(STDOUT) - ) - end - private_class_method :get_http_sender - - def self.get_udp_sender(encoder, address) - pair = address.split(":", 2) - host = pair[0] - port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT - - Jaeger::UdpSender.new( - host: host, - port: port, - encoder: encoder, - logger: Logger.new(STDOUT) - ) - end - private_class_method :get_udp_sender - end - end -end diff --git a/lib/gitlab/tracing/rack_middleware.rb b/lib/gitlab/tracing/rack_middleware.rb deleted file mode 100644 index e6a31293f7b..00000000000 --- a/lib/gitlab/tracing/rack_middleware.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require 'opentracing' - -module Gitlab - module Tracing - class RackMiddleware - include Common - - REQUEST_METHOD = 'REQUEST_METHOD' - - def initialize(app) - @app = app - end - - def call(env) - method = env[REQUEST_METHOD] - - context = tracer.extract(OpenTracing::FORMAT_RACK, env) - tags = { - 'component' => 'rack', - 'span.kind' => 'server', - 'http.method' => method, - 'http.url' => self.class.build_sanitized_url_from_env(env) - } - - in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span| - @app.call(env).tap do |status_code, _headers, _body| - span.set_tag('http.status_code', status_code) - end - end - end - - # Generate a sanitized (safe) request URL from the rack environment - def self.build_sanitized_url_from_env(env) - request = ActionDispatch::Request.new(env) - - original_url = request.original_url - uri = URI.parse(original_url) - uri.query = request.filtered_parameters.to_query if uri.query.present? - - uri.to_s - end - end - end -end diff --git a/lib/gitlab/tracing/rails/action_view_subscriber.rb b/lib/gitlab/tracing/rails/action_view_subscriber.rb deleted file mode 100644 index 88816e1fb32..00000000000 --- a/lib/gitlab/tracing/rails/action_view_subscriber.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracing - module Rails - class ActionViewSubscriber - include RailsCommon - - COMPONENT_TAG = 'ActionView' - RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view' - RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view' - RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view' - - # Instruments Rails ActionView events for opentracing. - # Returns a lambda, which, when called will unsubscribe from the notifications - def self.instrument - subscriber = new - - subscriptions = [ - ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| - subscriber.notify_render_template(start, finish, payload) - end, - ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| - subscriber.notify_render_collection(start, finish, payload) - end, - ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| - subscriber.notify_render_partial(start, finish, payload) - end - ] - - create_unsubscriber subscriptions - end - - # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html - def notify_render_template(start, finish, payload) - generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload)) - end - - def notify_render_collection(start, finish, payload) - generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload)) - end - - def notify_render_partial(start, finish, payload) - generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload)) - end - - private - - def tags_for_render_template(payload) - { - 'component' => COMPONENT_TAG, - 'template.id' => payload[:identifier], - 'template.layout' => payload[:layout] - } - end - - def tags_for_render_collection(payload) - { - 'component' => COMPONENT_TAG, - 'template.id' => payload[:identifier], - 'template.count' => payload[:count] || 0, - 'template.cache.hits' => payload[:cache_hits] || 0 - } - end - - def tags_for_render_partial(payload) - { - 'component' => COMPONENT_TAG, - 'template.id' => payload[:identifier] - } - end - end - end - end -end diff --git a/lib/gitlab/tracing/rails/active_record_subscriber.rb b/lib/gitlab/tracing/rails/active_record_subscriber.rb deleted file mode 100644 index 32f5658e57e..00000000000 --- a/lib/gitlab/tracing/rails/active_record_subscriber.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracing - module Rails - class ActiveRecordSubscriber - include RailsCommon - - ACTIVE_RECORD_NOTIFICATION_TOPIC = 'sql.active_record' - OPERATION_NAME_PREFIX = 'active_record:' - DEFAULT_OPERATION_NAME = 'sqlquery' - - # Instruments Rails ActiveRecord events for opentracing. - # Returns a lambda, which, when called will unsubscribe from the notifications - def self.instrument - subscriber = new - - subscription = ActiveSupport::Notifications.subscribe(ACTIVE_RECORD_NOTIFICATION_TOPIC) do |_, start, finish, _, payload| - subscriber.notify(start, finish, payload) - end - - create_unsubscriber [subscription] - end - - # For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html - def notify(start, finish, payload) - generate_span_for_notification(notification_name(payload), start, finish, payload, tags_for_notification(payload)) - end - - private - - def notification_name(payload) - OPERATION_NAME_PREFIX + (payload[:name].presence || DEFAULT_OPERATION_NAME) - end - - def tags_for_notification(payload) - { - 'component' => 'ActiveRecord', - 'span.kind' => 'client', - 'db.type' => 'sql', - 'db.connection_id' => payload[:connection_id], - 'db.cached' => payload[:cached] || false, - 'db.statement' => payload[:sql] - } - end - end - end - end -end diff --git a/lib/gitlab/tracing/rails/rails_common.rb b/lib/gitlab/tracing/rails/rails_common.rb deleted file mode 100644 index 88e914f62f8..00000000000 --- a/lib/gitlab/tracing/rails/rails_common.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracing - module Rails - module RailsCommon - extend ActiveSupport::Concern - include Gitlab::Tracing::Common - - class_methods do - def create_unsubscriber(subscriptions) - -> { subscriptions.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) } } - end - end - - def generate_span_for_notification(operation_name, start, finish, payload, tags) - exception = payload[:exception] - - postnotify_span(operation_name, start, finish, tags: tags, exception: exception) - end - end - end - end -end diff --git a/lib/gitlab/tracing/sidekiq/client_middleware.rb b/lib/gitlab/tracing/sidekiq/client_middleware.rb deleted file mode 100644 index 2b71c1ea21e..00000000000 --- a/lib/gitlab/tracing/sidekiq/client_middleware.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'opentracing' - -module Gitlab - module Tracing - module Sidekiq - class ClientMiddleware - include SidekiqCommon - - SPAN_KIND = 'client' - - def call(worker_class, job, queue, redis_pool) - in_tracing_span( - operation_name: "sidekiq:#{job['class']}", - tags: tags_from_job(job, SPAN_KIND)) do |span| - # Inject the details directly into the job - tracer.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, job) - - yield - end - end - end - end - end -end diff --git a/lib/gitlab/tracing/sidekiq/server_middleware.rb b/lib/gitlab/tracing/sidekiq/server_middleware.rb deleted file mode 100644 index 5b43c4310e6..00000000000 --- a/lib/gitlab/tracing/sidekiq/server_middleware.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'opentracing' - -module Gitlab - module Tracing - module Sidekiq - class ServerMiddleware - include SidekiqCommon - - SPAN_KIND = 'server' - - def call(worker, job, queue) - context = tracer.extract(OpenTracing::FORMAT_TEXT_MAP, job) - - in_tracing_span( - operation_name: "sidekiq:#{job['class']}", - child_of: context, - tags: tags_from_job(job, SPAN_KIND)) do |span| - yield - end - end - end - end - end -end diff --git a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb b/lib/gitlab/tracing/sidekiq/sidekiq_common.rb deleted file mode 100644 index a911a29d773..00000000000 --- a/lib/gitlab/tracing/sidekiq/sidekiq_common.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracing - module Sidekiq - module SidekiqCommon - include Gitlab::Tracing::Common - - def tags_from_job(job, kind) - { - 'component' => 'sidekiq', - 'span.kind' => kind, - 'sidekiq.queue' => job['queue'], - 'sidekiq.jid' => job['jid'], - 'sidekiq.retry' => job['retry'].to_s, - 'sidekiq.args' => job['args']&.join(", ") - } - end - end - end - end -end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f86d599e4cb..42cf1ec1f0e 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -30,6 +30,10 @@ module Gitlab snippet_url(object) when Milestone milestone_url(object) + when ::Ci::Build + project_job_url(object.project, object) + when User + user_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 75477c7cf18..08156d7ffa6 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -117,9 +117,11 @@ module Gitlab container_registry_enabled: Gitlab.config.registry.enabled, gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, + influxdb_metrics_enabled: Gitlab::Metrics.influx_metrics_enabled?, ldap_enabled: Gitlab.config.ldap.enabled, mattermost_enabled: Gitlab.config.mattermost.enabled, omniauth_enabled: Gitlab::Auth.omniauth_enabled?, + prometheus_metrics_enabled: Gitlab::Metrics.prometheus_metrics_enabled?, reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, signup_enabled: Gitlab::CurrentSettings.allow_signup? } diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index f0557f6ad68..ede60c9ab1d 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -7,7 +7,7 @@ module Gitlab class UserExtractor # Not using `Devise.email_regexp` to filter out any chars that an email # does not end with and not pinning the email to a start of end of a string. - EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/ + EMAIL_REGEXP = /(?<email>([^@\s]+@[^@\s]+(?<!\W)))/.freeze USERNAME_REGEXP = User.reference_pattern def initialize(text) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0c2acac3d1e..46a7b5b982a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -63,16 +63,34 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:, append_sha:) + def send_git_archive(repository, ref:, format:, append_sha:, path: nil) + path_enabled = Feature.enabled?(:git_archive_path, default_enabled: true) + path = nil unless path_enabled + format ||= 'tar.gz' format = format.downcase - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) - raise "Repository or ref not found" if params.empty? - params['GitalyServer'] = gitaly_server_hash(repository) + metadata = repository.archive_metadata( + ref, + Gitlab.config.gitlab.repository_downloads_path, + format, + append_sha: append_sha, + path: path + ) - # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. + raise "Repository or ref not found" if metadata.empty? + + params = + if path_enabled + send_git_archive_params(repository, metadata, path, archive_format(format)) + else + metadata + end + + # If present, DisableCache must be a Boolean. Otherwise + # workhorse ignores it. params['DisableCache'] = true if git_archive_cache_disabled? + params['GitalyServer'] = gitaly_server_hash(repository) [ SEND_DATA_HEADER, @@ -216,10 +234,19 @@ module Gitlab protected + # This is the outermost encoding of a senddata: header. It is safe for + # inclusion in HTTP response headers def encode(hash) Base64.urlsafe_encode64(JSON.dump(hash)) end + # This is for encoding individual fields inside the senddata JSON that + # contain binary data. In workhorse, the corresponding struct field should + # be type []byte + def encode_binary(binary) + Base64.encode64(binary) + end + def gitaly_server_hash(repository) { address: Gitlab::GitalyClient.address(repository.project.repository_storage), @@ -238,6 +265,34 @@ module Gitlab def git_archive_cache_disabled? ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) end + + def archive_format(format) + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + Gitaly::GetArchiveRequest::Format::TAR_BZ2 + when "tar" + Gitaly::GetArchiveRequest::Format::TAR + when "zip" + Gitaly::GetArchiveRequest::Format::ZIP + else + Gitaly::GetArchiveRequest::Format::TAR_GZ + end + end + + def send_git_archive_params(repository, metadata, path, format) + { + 'ArchivePath' => metadata['ArchivePath'], + 'GetArchiveRequest' => encode_binary( + Gitaly::GetArchiveRequest.new( + repository: repository.gitaly_repository, + commit_id: metadata['CommitId'], + prefix: metadata['ArchivePrefix'], + format: format, + path: path.presence || "" + ).to_proto + ) + } + end end end end diff --git a/lib/haml_lint/inline_javascript.rb b/lib/haml_lint/inline_javascript.rb index 2e98227a05e..1b17162f71d 100644 --- a/lib/haml_lint/inline_javascript.rb +++ b/lib/haml_lint/inline_javascript.rb @@ -1,4 +1,3 @@ -# rubocop:disable Naming/FileName # frozen_string_literal: true unless Rails.env.production? diff --git a/lib/peek/views/tracing.rb b/lib/peek/views/tracing.rb index 0de32a8fdda..94726a498ea 100644 --- a/lib/peek/views/tracing.rb +++ b/lib/peek/views/tracing.rb @@ -4,9 +4,9 @@ module Peek module Views class Tracing < View def results - { - tracing_url: Gitlab::Tracing.tracing_url - } + tracing_url = Labkit::Tracing.tracing_url(Gitlab.process_name) + + { tracing_url: tracing_url } end end end diff --git a/lib/quality/kubernetes_client.rb b/lib/quality/kubernetes_client.rb index 2ff9e811425..190b48ba7cb 100644 --- a/lib/quality/kubernetes_client.rb +++ b/lib/quality/kubernetes_client.rb @@ -18,6 +18,8 @@ module Quality 'delete', 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa', '--now', + '--ignore-not-found', + '--include-uninitialized', %(-l release="#{release_name}") ] diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb new file mode 100644 index 00000000000..4c8cb6e97cc --- /dev/null +++ b/lib/quality/seeders/issues.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# rubocop:disable CodeReuse/ActiveRecord +module Quality + module Seeders + class Issues + DEFAULT_BACKFILL_WEEKS = 52 + DEFAULT_AVERAGE_ISSUES_PER_WEEK = 10 + + attr_reader :project, :user + + def initialize(project:) + @project = project + end + + def seed(backfill_weeks: DEFAULT_BACKFILL_WEEKS, average_issues_per_week: DEFAULT_AVERAGE_ISSUES_PER_WEEK) + created_at = backfill_weeks.to_i.weeks.ago + team = project.team.users + created_issues_count = 0 + + loop do + rand(average_issues_per_week * 2).times do + params = { + title: FFaker::Lorem.sentence(6), + description: FFaker::Lorem.sentence, + created_at: created_at + rand(6).days, + state: %w[opened closed].sample, + milestone: project.milestones.sample, + assignee_ids: Array(team.pluck(:id).sample(3)), + labels: labels.join(',') + } + issue = ::Issues::CreateService.new(project, team.sample, params).execute + + if issue.persisted? + created_issues_count += 1 + print '.' # rubocop:disable Rails/Output + end + end + + created_at += 1.week + + break if created_at > Time.now + end + + created_issues_count + end + + private + + def labels + @labels_pool ||= project.labels.limit(rand(3)).pluck(:title).tap do |labels_array| + labels_array.concat(project.group.labels.limit(rand(3)).pluck(:title)) if project.group + end + end + end + end +end +# rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 7c3e7759dd0..467711fb74e 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.18.0') + @required_version ||= Gitlab::VersionInfo.parse('2.21.0') end def self.current_version diff --git a/lib/system_check/app/ruby_version_check.rb b/lib/system_check/app/ruby_version_check.rb index 60e07718338..53da62df176 100644 --- a/lib/system_check/app/ruby_version_check.rb +++ b/lib/system_check/app/ruby_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.new(2, 3, 5) + @required_version ||= Gitlab::VersionInfo.new(2, 5, 3) end def self.current_version diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 560a52053d8..8cf7c9e89f0 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -209,7 +209,7 @@ namespace :gemojione do image.destroy! end - EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i + EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i.freeze def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) # Rename file from unicode to emoji name matches = EMOJI_IMAGE_PATH_RE.match(image_path) diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake new file mode 100644 index 00000000000..155ba979b36 --- /dev/null +++ b/lib/tasks/gitlab/seed.rake @@ -0,0 +1,34 @@ +namespace :gitlab do + namespace :seed do + desc "GitLab | Seed | Seeds issues" + task :issues, [:project_full_path] => :environment do |t, args| + projects = + if args.project_full_path + project = Project.find_by_full_path(args.project_full_path) + + unless project + error_message = "Project '#{args.project_full_path}' does not exist!" + potential_projects = Project.search(args.project_full_path) + + if potential_projects.present? + error_message += " Did you mean '#{potential_projects.first.full_path}'?" + end + + puts error_message.color(:red) + exit 1 + end + + [project] + else + Project.find_each + end + + projects.each do |project| + puts "\nSeeding issues for the '#{project.full_path}' project" + seeder = Quality::Seeders::Issues.new(project: project) + issues_created = seeder.seed(backfill_weeks: 5, average_issues_per_week: 2) + puts "\n#{issues_created} issues created!" + end + end + end +end |