diff options
Diffstat (limited to 'lib')
28 files changed, 436 insertions, 209 deletions
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 8a54f7f3f05..7cdee8aced7 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -76,6 +76,27 @@ module API end end + desc 'Update an existing deploy key for a project' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + optional :title, type: String, desc: 'The name of the deploy key' + optional :can_push, type: Boolean, desc: "Can deploy key push to the project's repository" + at_least_one_of :title, :can_push + end + put ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys.find(params.delete(:key_id)) + + authorize!(:update_deploy_key, key) + + if key.update_attributes(declared_params(include_missing: false)) + present key, with: Entities::SSHKey + else + render_validation_error!(key) + end + end + desc 'Enable a deploy key for a project' do detail 'This feature was added in GitLab 8.11' success Entities::SSHKey diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ded5c65e303..a836df3dc81 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -226,7 +226,7 @@ module API end class ProjectSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at @@ -236,7 +236,7 @@ module API end class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name + expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at diff --git a/lib/api/groups.rb b/lib/api/groups.rb index e14a988a153..ebbaed0cbb7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -83,7 +83,7 @@ module API group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute if group.persisted? - present group, with: Entities::Group, current_user: current_user + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to save group #{group.errors.messages}", 400) end @@ -101,8 +101,6 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - at_least_one_of :name, :path, :description, :visibility, - :lfs_enabled, :request_access_enabled end put ':id' do group = find_group!(params[:id]) diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 98bc9c28527..64efe82a937 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -49,6 +49,7 @@ module API requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' requires :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' requires :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' @@ -77,6 +78,7 @@ module API optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' optional :code, type: String, desc: 'The content of the snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 82f513c984e..25027c3b114 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -110,6 +110,7 @@ module API optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do requires :metrics_host, type: String, desc: 'The InfluxDB host' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 53f5953a8fb..c630c24c339 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -58,6 +58,7 @@ module API requires :title, type: String, desc: 'The title of a snippet' requires :file_name, type: String, desc: 'The name of a snippet file' requires :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, default: 'internal', @@ -85,6 +86,7 @@ module API optional :title, type: String, desc: 'The title of a snippet' optional :file_name, type: String, desc: 'The name of a snippet file' optional :content, type: String, desc: 'The content of a snippet' + optional :description, type: String, desc: 'The description of a snippet' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the snippet' diff --git a/lib/api/users.rb b/lib/api/users.rb index 3f87a403a09..dda64715ee1 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -124,10 +124,6 @@ module API optional :name, type: String, desc: 'The name of the user' optional :username, type: String, desc: 'The username of the user' use :optional_attributes - at_least_one_of :email, :password, :name, :username, :skype, :linkedin, - :twitter, :website_url, :organization, :projects_limit, - :extern_uid, :provider, :bio, :location, :admin, - :can_create_group, :confirm, :external end put ":id" do authenticated_as_admin! diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b06474cda7f..22af2671b18 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -50,10 +50,23 @@ module Ci end end + def stage_seeds(pipeline) + trigger_request = pipeline.trigger_requests.first + + seeds = @stages.uniq.map do |stage| + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, trigger_request) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + def build_attributes(name) job = @jobs[name.to_sym] || {} - { - stage_idx: @stages.index(job[:stage]), + + { stage_idx: @stages.index(job[:stage]), stage: job[:stage], commands: job[:commands], tag_list: job[:tags] || [], @@ -71,8 +84,7 @@ module Ci dependencies: job[:dependencies], after_script: job[:after_script], environment: job[:environment] - }.compact - } + }.compact } end def self.validation_message(content) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 099c45dcfb7..da07ba2f2a3 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,8 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) + REGISTRY_SCOPES = [:read_registry].freeze + # Scopes used for GitLab API access API_SCOPES = [:api, :read_user].freeze @@ -11,8 +13,10 @@ module Gitlab # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze + AVAILABLE_SCOPES = (API_SCOPES + REGISTRY_SCOPES).freeze + # Other available scopes - OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze + OPTIONAL_SCOPES = (AVAILABLE_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) @@ -26,8 +30,8 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password) || oauth_access_token_check(login, password) || - user_with_password_for_git(login, password) || personal_access_token_check(password) || + user_with_password_for_git(login, password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) @@ -109,6 +113,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) @@ -121,17 +126,23 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) - if token && valid_api_token?(token) - Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) + if token && valid_scoped_token?(token, AVAILABLE_SCOPES.map(&:to_s)) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, abilities_for_scope(token.scopes)) end end def valid_oauth_token?(token) - token && token.accessible? && valid_api_token?(token) + token && token.accessible? && valid_scoped_token?(token, ["api"]) end - def valid_api_token?(token) - AccessTokenValidationService.new(token).include_any_scope?(['api']) + def valid_scoped_token?(token, scopes) + AccessTokenValidationService.new(token).include_any_scope?(scopes) + end + + def abilities_for_scope(scopes) + scopes.map do |scope| + self.public_send(:"#{scope}_scope_authentication_abilities") + end.flatten.uniq end def lfs_token_check(login, password) @@ -202,6 +213,16 @@ module Gitlab :create_container_image ] end + alias_method :api_scope_authentication_abilities, :full_authentication_abilities + + def read_registry_scope_authentication_abilities + [:read_container_image] + end + + # The currently used auth method doesn't allow any actions for this scope + def read_user_scope_authentication_abilities + [] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 39b86c61a18..75451cf8aa9 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -15,6 +15,10 @@ module Gitlab def success? actor.present? || type == :ci end + + def failed? + !success? + end end end end diff --git a/lib/gitlab/ci/stage/seed.rb b/lib/gitlab/ci/stage/seed.rb new file mode 100644 index 00000000000..f81f9347b4d --- /dev/null +++ b/lib/gitlab/ci/stage/seed.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + module Stage + class Seed + attr_reader :pipeline + delegate :project, to: :pipeline + + def initialize(pipeline, stage, jobs) + @pipeline = pipeline + @stage = { name: stage } + @jobs = jobs.to_a.dup + end + + def user=(current_user) + @jobs.map! do |attributes| + attributes.merge(user: current_user) + end + end + + def stage + @stage.merge(project: project) + end + + def builds + trigger = pipeline.trigger_requests.first + + @jobs.map do |attributes| + attributes.merge(project: project, + ref: pipeline.ref, + tag: pipeline.tag, + trigger_request: trigger) + end + end + + def create! + pipeline.stages.create!(stage).tap do |stage| + builds_attributes = builds.map do |attributes| + attributes.merge(stage_id: stage.id) + end + + pipeline.builds.create!(builds_attributes).each do |build| + yield build if block_given? + end + end + end + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 15992b77680..060e013183f 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -28,7 +28,7 @@ module Gitlab union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) - @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + @activity_dates = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| activities[event["date"]] += event["total_amount"] end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 182a30fd74d..e47fb85b5ee 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages_name, + stages: pipeline.stages_names, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 7948782aecc..371cbe04b9b 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -37,6 +37,16 @@ module Gitlab def complete? start_sha && head_sha end + + def compare_in(project) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + project.commit(head_sha) + else + straight = start_sha == base_sha + CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) + end + end end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 4d96778a2b2..f80afb20f0c 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -145,23 +145,9 @@ module Gitlab private def find_diff_file(repository) - # We're at the initial commit, so just get that as we can't compare to anything. - compare = - if Gitlab::Git.blank_ref?(start_sha) - Gitlab::Git::Commit.find(repository.raw_repository, head_sha) - else - Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ) - end - - diff = compare.diffs(paths: paths).first - - return unless diff + return unless diff_refs.complete? - Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) + diff_refs.compare_in(repository.project).diffs(paths: paths, expanded: true).diff_files.first end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index dcabb5f7fe5..b68a1636814 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -216,7 +216,7 @@ module Gitlab def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths) + compare.diffs(paths: paths, expanded: true) end def position(diff_file, old_line, new_line) diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 270d67dd50c..7f884183bb1 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -6,12 +6,13 @@ module Gitlab end def call(env) - route = Gitlab::EtagCaching::Router.match(env) + request = Rack::Request.new(env) + route = Gitlab::EtagCaching::Router.match(request) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(env) + etag, cached_value_present = get_etag(request) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -27,8 +28,8 @@ module Gitlab private - def get_etag(env) - cache_key = env['PATH_INFO'] + def get_etag(request) + cache_key = request.path store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index ca49eda51fb..dccc66b3918 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -53,8 +53,8 @@ module Gitlab ) ].freeze - def self.match(env) - ROUTES.find { |route| route.regexp.match(env['PATH_INFO']) } + def self.match(request) + ROUTES.find { |route| route.regexp.match(request.path_info) } end end end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index 696a2acd5e3..78e440395a5 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -3,7 +3,7 @@ module Gitlab class Compare attr_reader :head, :base, :straight - def initialize(repository, base, head, straight = false) + def initialize(repository, base, head, straight: false) @repository = repository @straight = straight diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb new file mode 100644 index 00000000000..b3c759b4730 --- /dev/null +++ b/lib/gitlab/health_checks/prometheus_text_format.rb @@ -0,0 +1,40 @@ +module Gitlab + module HealthChecks + class PrometheusTextFormat + def marshal(metrics) + "#{metrics_with_type_declarations(metrics).join("\n")}\n" + end + + private + + def metrics_with_type_declarations(metrics) + type_declaration_added = {} + + metrics.flat_map do |metric| + metric_lines = [] + + unless type_declaration_added.key?(metric.name) + type_declaration_added[metric.name] = true + metric_lines << metric_type_declaration(metric) + end + + metric_lines << metric_text(metric) + end + end + + def metric_type_declaration(metric) + "# TYPE #{metric.name} gauge" + end + + def metric_text(metric) + labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || '' + + if labels.empty? + "#{metric.name} #{metric.value}" + else + "#{metric.name}{#{labels}} #{metric.value}" + end + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index d0f3cf2b514..ff2b1d08c3c 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -38,6 +38,7 @@ project_tree: - notes: - :author - :events + - :stages - :statuses - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e23a4715f..695852526cb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,6 +3,7 @@ module Gitlab class RelationFactory OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', + stages: 'Ci::Stage', statuses: 'commit_status', triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index 2d5e47a6f3b..5e299e26c54 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -41,11 +41,6 @@ module Gitlab def update_user_attributes if persisted? - if auth_hash.has_email? - gl_user.skip_reconfirmation! - gl_user.email = auth_hash.email - end - # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } identity ||= gl_user.identities.build(provider: auth_hash.provider) @@ -55,10 +50,6 @@ module Gitlab # For an existing identity with no change in DN, this line changes nothing. identity.extern_uid = auth_hash.uid end - - gl_user.ldap_email = auth_hash.has_email? - - gl_user end def changed? @@ -69,6 +60,10 @@ module Gitlab ldap_config.block_auto_created_users end + def sync_email_from_provider? + true + end + def allowed? Gitlab::LDAP::Access.allowed?(gl_user) end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index cb8db2f1e9f..4779755bb22 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,161 +1,10 @@ module Gitlab module Metrics - extend Gitlab::CurrentSettings - - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def self.settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + extend Gitlab::Metrics::InfluxDb + extend Gitlab::Metrics::Prometheus def self.enabled? - settings[:enabled] || false - end - - def self.mri? - RUBY_ENGINE == 'ruby' - end - - def self.method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - @method_call_threshold ||= settings[:method_call_threshold] - end - - def self.pool - @pool - end - - def self.submit_metrics(metrics) - prepared = prepare_metrics(metrics) - - pool.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError - end - end - end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - - def self.prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys - - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) - end - end - - new_hash - end - end - - def self.escape_value(value) - value.to_s.gsub('=', '\\=') - end - - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def self.measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) * 1000.0 - cpu_time = cpu_stop - cpu_start - - trans.increment("#{name}_real_time", real_time) - trans.increment("#{name}_cpu_time", cpu_time) - trans.increment("#{name}_call_count", 1) - - retval - end - - # Adds a tag to the current transaction (if any) - # - # name - The name of the tag to add. - # value - The value of the tag. - def self.tag_transaction(name, value) - trans = current_transaction - - trans&.add_tag(name, value) - end - - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def self.action=(action) - trans = current_transaction - - trans&.action = action - end - - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def self.add_event(*args) - trans = current_transaction - - trans&.add_event(*args) - end - - # Returns the prefix to use for the name of a series. - def self.series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end - - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - if enabled? - @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client. - new(udp: { host: host, port: port }) - end + influx_metrics_enabled? || prometheus_metrics_enabled? end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb new file mode 100644 index 00000000000..3a39791edbf --- /dev/null +++ b/lib/gitlab/metrics/influx_db.rb @@ -0,0 +1,170 @@ +module Gitlab + module Metrics + module InfluxDb + extend Gitlab::CurrentSettings + extend self + + MUTEX = Mutex.new + private_constant :MUTEX + + def influx_metrics_enabled? + settings[:enabled] || false + end + + RAILS_ROOT = Rails.root.to_s + METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s + PATH_REGEX = /^#{RAILS_ROOT}\/?/ + + def settings + @settings ||= { + enabled: current_application_settings[:metrics_enabled], + pool_size: current_application_settings[:metrics_pool_size], + timeout: current_application_settings[:metrics_timeout], + method_call_threshold: current_application_settings[:metrics_method_call_threshold], + host: current_application_settings[:metrics_host], + port: current_application_settings[:metrics_port], + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 + } + end + + def mri? + RUBY_ENGINE == 'ruby' + end + + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + @method_call_threshold ||= settings[:method_call_threshold] + end + + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end + end + end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) + end + + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def escape_value(value) + value.to_s.gsub('=', '\\=') + end + + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) * 1000.0 + cpu_time = cpu_stop - cpu_start + + trans.increment("#{name}_real_time", real_time) + trans.increment("#{name}_cpu_time", cpu_time) + trans.increment("#{name}_call_count", 1) + + retval + end + + # Adds a tag to the current transaction (if any) + # + # name - The name of the tag to add. + # value - The value of the tag. + def tag_transaction(name, value) + trans = current_transaction + + trans&.add_tag(name, value) + end + + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction + + trans&.action = action + end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction + + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client. + new(udp: { host: host, port: port }) + end + end + end + @pool + end + end + end + end +end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb new file mode 100644 index 00000000000..3b5a2907195 --- /dev/null +++ b/lib/gitlab/metrics/null_metric.rb @@ -0,0 +1,10 @@ +module Gitlab + module Metrics + # Mocks ::Prometheus::Client::Metric and all derived metrics + class NullMetric + def method_missing(name, *args, &block) + nil + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb new file mode 100644 index 00000000000..60686509332 --- /dev/null +++ b/lib/gitlab/metrics/prometheus.rb @@ -0,0 +1,41 @@ +require 'prometheus/client' + +module Gitlab + module Metrics + module Prometheus + include Gitlab::CurrentSettings + + def prometheus_metrics_enabled? + @prometheus_metrics_enabled ||= current_application_settings[:prometheus_metrics_enabled] || false + end + + def registry + @registry ||= ::Prometheus::Client.registry + end + + def counter(name, docstring, base_labels = {}) + provide_metric(name) || registry.counter(name, docstring, base_labels) + end + + def summary(name, docstring, base_labels = {}) + provide_metric(name) || registry.summary(name, docstring, base_labels) + end + + def gauge(name, docstring, base_labels = {}) + provide_metric(name) || registry.gauge(name, docstring, base_labels) + end + + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets) + end + + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.new + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index afd24b4dcc5..7307f8c2c87 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -12,6 +12,7 @@ module Gitlab def initialize(auth_hash) self.auth_hash = auth_hash + update_email end def persisted? @@ -174,6 +175,22 @@ module Gitlab } end + def sync_email_from_provider? + auth_hash.provider.to_s == Gitlab.config.omniauth.sync_email_from_provider.to_s + end + + def update_email + if auth_hash.has_email? && sync_email_from_provider? + if persisted? + gl_user.skip_reconfirmation! + gl_user.email = auth_hash.email + end + + gl_user.external_email = true + gl_user.email_provider = auth_hash.provider + end + end + def log Gitlab::AppLogger end |