diff options
Diffstat (limited to 'lib/gitlab')
| -rw-r--r-- | lib/gitlab/auth.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/daemon.rb | 62 | ||||
| -rw-r--r-- | lib/gitlab/environment.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/git/commit.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/git/repository.rb | 124 | ||||
| -rw-r--r-- | lib/gitlab/git/storage.rb | 22 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/circuit_breaker.rb | 144 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/forked_storage_check.rb | 55 | ||||
| -rw-r--r-- | lib/gitlab/git/storage/health.rb | 91 | ||||
| -rw-r--r-- | lib/gitlab/health_checks/fs_shards_check.rb | 21 | ||||
| -rw-r--r-- | lib/gitlab/i18n.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/metrics/base_sampler.rb | 75 | ||||
| -rw-r--r-- | lib/gitlab/metrics/sidekiq_metrics_exporter.rb | 39 | ||||
| -rw-r--r-- | lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/prometheus/queries/environment_query.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/usage_data.rb | 4 |
16 files changed, 496 insertions, 160 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 9bed81e7327..7d3aa532750 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -218,7 +218,8 @@ module Gitlab def full_authentication_abilities read_authentication_abilities + [ :push_code, - :create_container_image + :create_container_image, + :admin_container_image ] end alias_method :api_scope_authentication_abilities, :full_authentication_abilities diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb new file mode 100644 index 00000000000..dfd17e35707 --- /dev/null +++ b/lib/gitlab/daemon.rb @@ -0,0 +1,62 @@ +module Gitlab + class Daemon + def self.initialize_instance(*args) + raise "#{name} singleton instance already initialized" if @instance + @instance = new(*args) + Kernel.at_exit(&@instance.method(:stop)) + @instance + end + + def self.instance + @instance ||= initialize_instance + end + + attr_reader :thread + + def thread? + !thread.nil? + end + + def initialize + @mutex = Mutex.new + end + + def enabled? + true + end + + def start + return unless enabled? + + @mutex.synchronize do + return thread if thread? + + @thread = Thread.new { start_working } + end + end + + def stop + @mutex.synchronize do + return unless thread? + + stop_working + + if thread + thread.wakeup if thread.alive? + thread.join + @thread = nil + end + end + end + + private + + def start_working + raise NotImplementedError + end + + def stop_working + # no-ops + end + end +end diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb new file mode 100644 index 00000000000..5e0dd6e7859 --- /dev/null +++ b/lib/gitlab/environment.rb @@ -0,0 +1,7 @@ +module Gitlab + module Environment + def self.hostname + @hostname ||= ENV['HOSTNAME'] || Socket.gethostname + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index ca7e3a7c4be..600d886e818 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -102,7 +102,7 @@ module Gitlab if is_enabled repo.gitaly_commit_client.between(base, head) else - repo.commits_between(base, head).map { |c| decorate(c) } + repo.rugged_commits_between(base, head).map { |c| decorate(c) } end end rescue Rugged::ReferenceError diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 734aed8fbc1..f6f9d49bf37 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -58,17 +58,18 @@ module Gitlab end end - # Alias to old method for compatibility - def raw - rugged - end - def rugged - @rugged ||= Rugged::Repository.new(path, alternates: alternate_object_directories) + @rugged ||= circuit_breaker.perform do + Rugged::Repository.new(path, alternates: alternate_object_directories) + end rescue Rugged::RepositoryError, Rugged::OSError raise NoRepository.new('no repository for such path') end + def circuit_breaker + @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) + end + # Returns an Array of branch names # sorted by name ASC def branch_names @@ -296,7 +297,23 @@ module Gitlab # after: Time.new(2016, 4, 21, 14, 32, 10) # ) # + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/446 def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + raw_log(options).map { |c| Commit.decorate(c) } end @@ -324,7 +341,9 @@ module Gitlab # Return a collection of Rugged::Commits between the two revspec arguments. # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for # a detailed list of valid arguments. - def commits_between(from, to) + # + # Gitaly note: JV: to be deprecated in favor of Commit.between + def rugged_commits_between(from, to) walker = Rugged::Walker.new(rugged) walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) @@ -708,20 +727,6 @@ module Gitlab end def raw_log(options) - default_options = { - limit: 10, - offset: 0, - path: nil, - follow: false, - skip_merges: false, - disable_walk: false, - after: nil, - before: nil - } - - options = default_options.merge(options) - options[:limit] ||= 0 - options[:offset] ||= 0 actual_ref = options[:ref] || root_ref begin sha = sha_from_ref(actual_ref) @@ -857,46 +862,6 @@ module Gitlab submodule_data.select { |path, data| data['id'] } end - # Returns true if +commit+ introduced changes to +path+, using commit - # trees to make that determination. Uses the history simplification - # rules that `git log` uses by default, where a commit is omitted if it - # is TREESAME to any parent. - # - # If the +follow+ option is true and the file specified by +path+ was - # renamed, then the path value is set to the old path. - def commit_touches_path?(commit, path, follow, walker) - entry = tree_entry(commit, path) - - if commit.parents.empty? - # This is the root commit, return true if it has +path+ in its tree - return !entry.nil? - end - - num_treesame = 0 - commit.parents.each do |parent| - parent_entry = tree_entry(parent, path) - - # Only follow the first TREESAME parent for merge commits - if num_treesame > 0 - walker.hide(parent) - next - end - - if entry.nil? && parent_entry.nil? - num_treesame += 1 - elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] - num_treesame += 1 - end - end - - case num_treesame - when 0 - detect_rename(commit, commit.parents.first, path) if follow - true - else false - end - end - # Find the entry for +path+ in the tree for +commit+ def tree_entry(commit, path) pathname = Pathname.new(path) @@ -924,43 +889,6 @@ module Gitlab tmp_entry end - # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was - # renamed in +commit+, then set +path+ to the old filename. - def detect_rename(commit, parent, path) - diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) - - # If +path+ is a filename, not a directory, then we should only have - # one delta. We don't need to follow renames for directories. - return nil if diff.each_delta.count > 1 - - delta = diff.each_delta.first - if delta.added? - full_diff = parent.diff(commit) - full_diff.find_similar! - - full_diff.each_delta do |full_delta| - if full_delta.renamed? && path == full_delta.new_file[:path] - # Look for the old path in ancestors - path.replace(full_delta.old_file[:path]) - end - end - end - end - - # Returns true if the index entry has the special file mode that denotes - # a submodule. - def submodule?(index_entry) - index_entry[:mode] == 57344 - end - - # Return a Rugged::Index that has read from the tree at +ref_name+ - def populated_index(ref_name) - commit = rev_parse_target(ref_name) - index = rugged.index - index.read_tree(commit.tree) - index - end - # Return the Rugged patches for the diff between +from+ and +to+. def diff_patches(from, to, options = {}, *paths) options ||= {} diff --git a/lib/gitlab/git/storage.rb b/lib/gitlab/git/storage.rb new file mode 100644 index 00000000000..e28be4b8a38 --- /dev/null +++ b/lib/gitlab/git/storage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Git + module Storage + class Inaccessible < StandardError + attr_reader :retry_after + + def initialize(message = nil, retry_after = nil) + super(message) + @retry_after = retry_after + end + end + + CircuitOpen = Class.new(Inaccessible) + + REDIS_KEY_PREFIX = 'storage_accessible:'.freeze + + def self.redis + Gitlab::Redis::SharedState + end + end + end +end diff --git a/lib/gitlab/git/storage/circuit_breaker.rb b/lib/gitlab/git/storage/circuit_breaker.rb new file mode 100644 index 00000000000..9ea9367d4b7 --- /dev/null +++ b/lib/gitlab/git/storage/circuit_breaker.rb @@ -0,0 +1,144 @@ +module Gitlab + module Git + module Storage + class CircuitBreaker + FailureInfo = Struct.new(:last_failure, :failure_count) + + attr_reader :storage, + :hostname, + :storage_path, + :failure_count_threshold, + :failure_wait_time, + :failure_reset_time, + :storage_timeout + + delegate :last_failure, :failure_count, to: :failure_info + + def self.reset_all! + pattern = "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}*" + + Gitlab::Git::Storage.redis.with do |redis| + all_storage_keys = redis.keys(pattern) + redis.del(*all_storage_keys) unless all_storage_keys.empty? + end + + RequestStore.delete(:circuitbreaker_cache) + end + + def self.for_storage(storage) + cached_circuitbreakers = RequestStore.fetch(:circuitbreaker_cache) do + Hash.new do |hash, storage_name| + hash[storage_name] = new(storage_name) + end + end + + cached_circuitbreakers[storage] + end + + def initialize(storage, hostname = Gitlab::Environment.hostname) + @storage = storage + @hostname = hostname + + config = Gitlab.config.repositories.storages[@storage] + @storage_path = config['path'] + @failure_count_threshold = config['failure_count_threshold'] + @failure_wait_time = config['failure_wait_time'] + @failure_reset_time = config['failure_reset_time'] + @storage_timeout = config['storage_timeout'] + end + + def perform + return yield unless Feature.enabled?('git_storage_circuit_breaker') + + check_storage_accessible! + + yield + end + + def circuit_broken? + return false if no_failures? + + recent_failure = last_failure > failure_wait_time.seconds.ago + too_many_failures = failure_count > failure_count_threshold + + recent_failure || too_many_failures + end + + # Memoizing the `storage_available` call means we only do it once per + # request when the storage is available. + # + # When the storage appears not available, and the memoized value is `false` + # we might want to try again. + def storage_available? + return @storage_available if @storage_available + + if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck + .storage_available?(storage_path, storage_timeout) + track_storage_accessible + else + track_storage_inaccessible + end + + @storage_available + end + + def check_storage_accessible! + if circuit_broken? + raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time) + end + + unless storage_available? + raise Gitlab::Git::Storage::Inaccessible.new("#{storage} not accessible", failure_wait_time) + end + end + + def no_failures? + last_failure.blank? && failure_count == 0 + end + + def track_storage_inaccessible + @failure_info = FailureInfo.new(Time.now, failure_count + 1) + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :last_failure, last_failure.to_i) + redis.hincrby(cache_key, :failure_count, 1) + redis.expire(cache_key, failure_reset_time) + end + end + end + + def track_storage_accessible + return if no_failures? + + @failure_info = FailureInfo.new(nil, 0) + + Gitlab::Git::Storage.redis.with do |redis| + redis.pipelined do + redis.hset(cache_key, :last_failure, nil) + redis.hset(cache_key, :failure_count, 0) + end + end + end + + def failure_info + @failure_info ||= get_failure_info + end + + def get_failure_info + last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis| + redis.hmget(cache_key, :last_failure, :failure_count) + end + + last_failure = Time.at(last_failure.to_i) if last_failure.present? + + FailureInfo.new(last_failure, failure_count.to_i) + end + + def cache_key + @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}" + end + end + end + end +end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb new file mode 100644 index 00000000000..91d8241f17b --- /dev/null +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -0,0 +1,55 @@ +module Gitlab + module Git + module Storage + module ForkedStorageCheck + extend self + + def storage_available?(path, timeout_seconds = 5) + status = timeout_check(path, timeout_seconds) + + status.success? + end + + def timeout_check(path, timeout_seconds) + filesystem_check_pid = check_filesystem_in_process(path) + + deadline = timeout_seconds.seconds.from_now.utc + wait_time = 0.01 + status = nil + + while status.nil? + if deadline > Time.now.utc + sleep(wait_time) + _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) + else + Process.kill('KILL', filesystem_check_pid) + # Blocking wait, so we are sure the process is gone before continuing + _pid, status = Process.wait2(filesystem_check_pid) + end + end + + status + end + + # This will spawn a new 2 processes to do the check: + # The outer child (waiter) will spawn another child process (stater). + # + # The stater is the process is performing the actual filesystem check + # the check might hang if the filesystem is acting up. + # In this case we will send a `KILL` to the waiter, which will still + # be responsive while the stater is hanging. + def check_filesystem_in_process(path) + spawn('ruby', '-e', ruby_check, path, [:out, :err] => '/dev/null') + end + + def ruby_check + <<~RUBY_FILESYSTEM_CHECK + inner_pid = fork { File.stat(ARGV.first) } + Process.waitpid(inner_pid) + exit $?.exitstatus + RUBY_FILESYSTEM_CHECK + end + end + end + end +end diff --git a/lib/gitlab/git/storage/health.rb b/lib/gitlab/git/storage/health.rb new file mode 100644 index 00000000000..2d723147f4f --- /dev/null +++ b/lib/gitlab/git/storage/health.rb @@ -0,0 +1,91 @@ +module Gitlab + module Git + module Storage + class Health + attr_reader :storage_name, :info + + def self.pattern_for_storage(storage_name) + "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage_name}:*" + end + + def self.for_all_storages + storage_names = Gitlab.config.repositories.storages.keys + results_per_storage = nil + + Gitlab::Git::Storage.redis.with do |redis| + keys_per_storage = all_keys_for_storages(storage_names, redis) + results_per_storage = load_for_keys(keys_per_storage, redis) + end + + results_per_storage.map do |name, info| + info.each { |i| i[:failure_count] = i[:failure_count].value.to_i } + new(name, info) + end + end + + def self.all_keys_for_storages(storage_names, redis) + keys_per_storage = {} + + redis.pipelined do + storage_names.each do |storage_name| + pattern = pattern_for_storage(storage_name) + + keys_per_storage[storage_name] = redis.keys(pattern) + end + end + + keys_per_storage + end + + def self.load_for_keys(keys_per_storage, redis) + info_for_keys = {} + + redis.pipelined do + keys_per_storage.each do |storage_name, keys_future| + info_for_storage = keys_future.value.map do |key| + { name: key, failure_count: redis.hget(key, :failure_count) } + end + + info_for_keys[storage_name] = info_for_storage + end + end + + info_for_keys + end + + def self.for_failing_storages + for_all_storages.select(&:failing?) + end + + def initialize(storage_name, info) + @storage_name = storage_name + @info = info + end + + def failing_info + @failing_info ||= info.select { |info_for_host| info_for_host[:failure_count] > 0 } + end + + def failing? + failing_info.any? + end + + def failing_on_hosts + @failing_on_hosts ||= failing_info.map do |info_for_host| + info_for_host[:name].split(':').last + end + end + + def failing_circuit_breakers + @failing_circuit_breakers ||= failing_on_hosts.map do |hostname| + CircuitBreaker.new(storage_name, hostname) + end + end + + def total_failures + @total_failures ||= failing_info.sum { |info_for_host| info_for_host[:failure_count] } + end + end + end + end +end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index 9e91c135956..eef97f54962 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -10,7 +10,9 @@ module Gitlab def readiness repository_storages.map do |storage_name| begin - if !storage_stat_test(storage_name) + if !storage_circuitbreaker_test(storage_name) + HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) + elsif !storage_stat_test(storage_name) HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) else with_temp_file(storage_name) do |tmp_file_path| @@ -36,7 +38,8 @@ module Gitlab [ storage_stat_metrics(storage_name), storage_write_metrics(storage_name), - storage_read_metrics(storage_name) + storage_read_metrics(storage_name), + storage_circuitbreaker_metrics(storage_name) ].flatten end end @@ -121,6 +124,12 @@ module Gitlab file_contents == RANDOM_STRING end + def storage_circuitbreaker_test(storage_name) + Gitlab::Git::Storage::CircuitBreaker.new(storage_name).perform { "OK" } + rescue Gitlab::Git::Storage::Inaccessible + nil + end + def storage_stat_metrics(storage_name) operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do with_timing { storage_stat_test(storage_name) } @@ -143,6 +152,14 @@ module Gitlab end end end + + def storage_circuitbreaker_metrics(storage_name) + operation_metrics(:filesystem_circuitbreaker, + :filesystem_circuitbreaker_latency_seconds, + shard: storage_name) do + with_timing { storage_circuitbreaker_test(storage_name) } + end + end end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index cc282d1415b..5d106b5c075 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -16,7 +16,8 @@ module Gitlab 'eo' => 'Esperanto', 'it' => 'Italiano', 'uk' => 'Українська', - 'ja' => '日本語' + 'ja' => '日本語', + 'ko' => '한국어' }.freeze def available_locales diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb index 219accfc029..716d20bb91a 100644 --- a/lib/gitlab/metrics/base_sampler.rb +++ b/lib/gitlab/metrics/base_sampler.rb @@ -1,20 +1,7 @@ require 'logger' module Gitlab module Metrics - class BaseSampler - def self.initialize_instance(*args) - raise "#{name} singleton instance already initialized" if @instance - @instance = new(*args) - at_exit(&@instance.method(:stop)) - @instance - end - - def self.instance - @instance - end - - attr_reader :running - + class BaseSampler < Daemon # interval - The sampling interval in seconds. def initialize(interval) interval_half = interval.to_f / 2 @@ -22,44 +9,7 @@ module Gitlab @interval = interval @interval_steps = (-interval_half..interval_half).step(0.1).to_a - @mutex = Mutex.new - end - - def enabled? - true - end - - def start - return unless enabled? - - @mutex.synchronize do - return if running - @running = true - - @thread = Thread.new do - sleep(sleep_interval) - - while running - safe_sample - - sleep(sleep_interval) - end - end - end - end - - def stop - @mutex.synchronize do - return unless running - - @running = false - - if @thread - @thread.wakeup if @thread.alive? - @thread.join - @thread = nil - end - end + super() end def safe_sample @@ -81,7 +31,7 @@ module Gitlab # potentially missing anything that happens in between samples). # 2. Don't sample data at the same interval two times in a row. def sleep_interval - while step = @interval_steps.sample + while (step = @interval_steps.sample) if step != @last_step @last_step = step @@ -89,6 +39,25 @@ module Gitlab end end end + + private + + attr_reader :running + + def start_working + @running = true + sleep(sleep_interval) + + while running + safe_sample + + sleep(sleep_interval) + end + end + + def stop_working + @running = false + end end end end diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb new file mode 100644 index 00000000000..5980a4ded2b --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -0,0 +1,39 @@ +require 'webrick' +require 'prometheus/client/rack/exporter' + +module Gitlab + module Metrics + class SidekiqMetricsExporter < Daemon + def enabled? + Gitlab::Metrics.metrics_folder_present? && settings.enabled + end + + def settings + Settings.monitoring.sidekiq_exporter + end + + private + + attr_reader :server + + def start_working + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + server.mount "/", Rack::Handler::WEBrick, rack_app + server.start + end + + def stop_working + server.shutdown + @server = nil + end + + def rack_app + Rack::Builder.app do + use Rack::Deflater + use ::Prometheus::Client::Rack::Exporter + run -> (env) { [404, {}, ['']] } + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb index db4708b22e4..32fe8201a8d 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -5,7 +5,7 @@ module Gitlab include QueryAdditionalMetrics def query(environment_id) - Environment.find_by(id: environment_id).try do |environment| + ::Environment.find_by(id: environment_id).try do |environment| query_metrics( common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f) ) diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 66f29d95177..1d17d3cfd56 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -3,7 +3,7 @@ module Gitlab module Queries class EnvironmentQuery < BaseQuery def query(environment_id) - Environment.find_by(id: environment_id).try do |environment| + ::Environment.find_by(id: environment_id).try do |environment| environment_slug = environment.slug timeframe_start = 8.hours.ago.to_f timeframe_end = Time.now.to_f diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e0ac21305a5..748e0a29184 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -27,8 +27,8 @@ module Gitlab ci_pipeline_schedules: ::Ci::PipelineSchedule.count, deploy_keys: DeployKey.count, deployments: Deployment.count, - environments: Environment.count, - in_review_folder: Environment.in_review_folder.count, + environments: ::Environment.count, + in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, keys: Key.count, |