From 175b4fa261259ab0d033482d10bb4159fee8e538 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 11 Dec 2019 18:08:10 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/api/keys.rb | 17 +++++++ lib/gitlab.rb | 4 +- lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml | 21 +++++--- lib/gitlab/cluster/lifecycle_events.rb | 6 +-- lib/gitlab/database/sha256_attribute.rb | 33 +++++++++++++ lib/gitlab/gitaly_client.rb | 8 +-- lib/gitlab/gpg.rb | 2 +- lib/gitlab/health_checks/puma_check.rb | 2 +- lib/gitlab/health_checks/unicorn_check.rb | 2 +- lib/gitlab/highlight.rb | 2 +- lib/gitlab/insecure_key_fingerprint.rb | 5 ++ lib/gitlab/metrics/influx_db.rb | 2 +- lib/gitlab/metrics/samplers/influx_sampler.rb | 6 +-- lib/gitlab/metrics/samplers/unicorn_sampler.rb | 2 +- lib/gitlab/redis/wrapper.rb | 4 +- lib/gitlab/runtime.rb | 62 ++++++++++++++++++++++++ lib/prometheus/pid_provider.rb | 6 +-- 17 files changed, 149 insertions(+), 35 deletions(-) create mode 100644 lib/gitlab/database/sha256_attribute.rb create mode 100644 lib/gitlab/runtime.rb (limited to 'lib') diff --git a/lib/api/keys.rb b/lib/api/keys.rb index d5280a0035d..8f2fd8cbae2 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -16,6 +16,23 @@ module API present key, with: Entities::SSHKeyWithUser, current_user: current_user end + + desc 'Get SSH Key information' do + success Entities::UserWithAdmin + end + params do + requires :fingerprint, type: String, desc: 'Search for a SSH fingerprint' + end + get do + authenticated_with_full_private_access! + + key = KeysFinder.new(current_user, params).execute + + not_found!('Key') unless key + present key, with: Entities::SSHKeyWithUser, current_user: current_user + rescue KeysFinder::InvalidFingerprint + render_api_error!('Failed to return the key', 400) + end end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 0e6db54eb46..f2bff51df38 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -100,8 +100,8 @@ module Gitlab end def self.process_name - return 'sidekiq' if Sidekiq.server? - return 'console' if defined?(Rails::Console) + return 'sidekiq' if Gitlab::Runtime.sidekiq? + return 'console' if Gitlab::Runtime.console? return 'test' if Rails.env.test? 'web' diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 9a3ecd1c34f..975cb3b7698 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -1,5 +1,16 @@ -# Full project: https://gitlab.com/pages/hugo -image: dettmering/hugo-build +--- +# All available Hugo versions are listed here: +# https://gitlab.com/pages/hugo/container_registry +image: registry.gitlab.com/pages/hugo:latest + +variables: + GIT_SUBMODULE_STRATEGY: recursive + +test: + script: + - hugo + except: + - master pages: script: @@ -9,9 +20,3 @@ pages: - public only: - master - -test: - script: - - hugo - except: - - master diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 2b3dc94fc5e..4ae75e0db0a 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -149,10 +149,10 @@ module Gitlab def in_clustered_environment? # Sidekiq doesn't fork - return false if Sidekiq.server? + return false if Gitlab::Runtime.sidekiq? # Unicorn always forks - return true if defined?(::Unicorn) + return true if Gitlab::Runtime.unicorn? # Puma sometimes forks return true if in_clustered_puma? @@ -162,7 +162,7 @@ module Gitlab end def in_clustered_puma? - return false unless defined?(::Puma) + return false unless Gitlab::Runtime.puma? @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 end diff --git a/lib/gitlab/database/sha256_attribute.rb b/lib/gitlab/database/sha256_attribute.rb new file mode 100644 index 00000000000..adf3f7fb5a6 --- /dev/null +++ b/lib/gitlab/database/sha256_attribute.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Class for casting binary data to hexadecimal SHA256 hashes (and vice-versa). + # + # Using Sha256Attribute allows you to store SHA256 values as binary while still + # using them as if they were stored as string values. This gives you the + # ease of use of string values, but without the storage overhead. + class Sha256Attribute < ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + # Casts binary data to a SHA256 and remove trailing = and newline from encode64 + def deserialize(value) + value = super(value) + if value.present? + Base64.encode64(value).delete("=").chomp("\n") + else + nil + end + end + + # Casts a SHA256 in a proper binary format. which is 32 bytes long + def serialize(value) + arg = if value.present? + Base64.decode64(value) + else + nil + end + + super(arg) + end + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5b47853b9c1..373539f5516 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -29,7 +29,7 @@ module Gitlab PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 - CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze + CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze GITALY_METADATA_FILENAME = '.gitaly-metadata' MUTEX = Mutex.new @@ -383,17 +383,13 @@ module Gitlab end def self.long_timeout - if web_app_server? + if Gitlab::Runtime.app_server? default_timeout else 6.hours end end - def self.web_app_server? - defined?(::Unicorn) || defined?(::Puma) - end - def self.storage_metadata_file_path(storage) Gitlab::GitalyClient::StorageSettings.allow_disk_access do File.join( diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index 829e64b11a4..abe90bba19c 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -135,7 +135,7 @@ module Gitlab end def cleanup_time - Sidekiq.server? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S + Gitlab::Runtime.sidekiq? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S end def tmp_keychains_created diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb index 7aafe29fbae..9f09070a57d 100644 --- a/lib/gitlab/health_checks/puma_check.rb +++ b/lib/gitlab/health_checks/puma_check.rb @@ -18,7 +18,7 @@ module Gitlab end def check - return unless defined?(::Puma) + return unless Gitlab::Runtime.puma? stats = Puma.stats stats = JSON.parse(stats) diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb index a30ae015257..cdc6d2a7519 100644 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -30,7 +30,7 @@ module Gitlab # to change so we can cache the list of servers. def http_servers strong_memoize(:http_servers) do - next unless defined?(::Unicorn::HttpServer) + next unless Gitlab::Runtime.unicorn? ObjectSpace.each_object(::Unicorn::HttpServer).to_a end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 381f1dd4e55..5663b9f20cf 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -68,7 +68,7 @@ module Gitlab end def timeout_time - Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND + Gitlab::Runtime.sidekiq? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND end def link_dependencies(text, highlighted_text) diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index e4f0e9d2c73..7b1cf5e7931 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -10,6 +10,7 @@ module Gitlab # class InsecureKeyFingerprint attr_accessor :key + alias_attribute :fingerprint_md5, :fingerprint # # Gets the base64 encoded string representing a rsa or dsa key @@ -21,5 +22,9 @@ module Gitlab def fingerprint OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') end + + def fingerprint_sha256 + Digest::SHA256.base64digest(Base64.decode64(@key)).scan(/../).join('').delete("=") + end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 269d90fa971..1f252572461 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -150,7 +150,7 @@ module Gitlab # Returns the prefix to use for the name of a series. def series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' end # Allow access from other metrics related middlewares diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 1eae0a7bf45..4e16e335bee 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -39,14 +39,10 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_' @metrics << Metric.new("#{prefix}#{series}", values, tags) end - - def sidekiq? - Sidekiq.server? - end end end end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 355f938704e..8c4d150adad 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -61,7 +61,7 @@ module Gitlab # it takes around 80ms. The instances of HttpServers are not a subject # to change so we can cache the list of servers. def http_servers - return [] unless defined?(::Unicorn::HttpServer) + return [] unless Gitlab::Runtime.unicorn? @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 412d00c6939..beceed3fa75 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -22,10 +22,10 @@ module Gitlab def pool_size # heuristic constant 5 should be a config setting somewhere -- related to CPU count? size = 5 - if Sidekiq.server? + if Gitlab::Runtime.sidekiq? # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] - elsif defined?(::Puma) + elsif Gitlab::Runtime.puma? size += Puma.cli_config.options[:max_threads] end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb new file mode 100644 index 00000000000..07a3afb8834 --- /dev/null +++ b/lib/gitlab/runtime.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + # Provides routines to identify the current runtime as which the application + # executes, such as whether it is an application server and which one. + module Runtime + class << self + def name + matches = [] + matches << :puma if puma? + matches << :unicorn if unicorn? + matches << :console if console? + matches << :sidekiq if sidekiq? + + raise "Ambiguous process match: #{matches}" if matches.size > 1 + + matches.first || :unknown + end + + def puma? + !!(defined?(::Puma) && bin == 'puma') + end + + # For unicorn, we need to check for actual server instances to avoid false positives. + def unicorn? + !!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer)) + end + + def sidekiq? + !!(defined?(::Sidekiq) && Sidekiq.server? && bin == 'sidekiq') + end + + def console? + !!defined?(::Rails::Console) + end + + def app_server? + puma? || unicorn? + end + + def multi_threaded? + puma? || sidekiq? + end + + private + + # Some example values from my system: + # puma: /data/cache/bundle-2.5/bin/puma + # unicorn: unicorn_rails master -E development -c /tmp/unicorn.rb -l 0.0.0.0:8080 + # sidekiq: /data/cache/bundle-2.5/bin/sidekiq + # thin: bin/rails + # console: bin/rails + def script_name + $0 + end + + def bin + File.basename(script_name) + end + end + end +end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb index 228639357ac..32beeb0d31e 100644 --- a/lib/prometheus/pid_provider.rb +++ b/lib/prometheus/pid_provider.rb @@ -5,11 +5,11 @@ module Prometheus extend self def worker_id - if Sidekiq.server? + if Gitlab::Runtime.sidekiq? sidekiq_worker_id - elsif defined?(Unicorn::Worker) + elsif Gitlab::Runtime.unicorn? unicorn_worker_id - elsif defined?(::Puma) + elsif Gitlab::Runtime.puma? puma_worker_id else unknown_process_id -- cgit v1.2.3