diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/banzai/filter/commit_reference_filter.rb | 2 | ||||
| -rw-r--r-- | lib/banzai/filter/relative_link_filter.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/checks/diff_check.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/data_builder/push.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/database.rb | 32 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client.rb | 27 | ||||
| -rw-r--r-- | lib/gitlab/optimistic_locking.rb | 1 | ||||
| -rw-r--r-- | lib/gitlab/shell.rb | 2 |
8 files changed, 57 insertions, 13 deletions
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index c3e5ac41cb8..d2880cb7e1d 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -23,7 +23,7 @@ module Banzai if project && project.valid_repo? # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/43894 - Gitlab::GitalyClient.allow_n_plus_1_calls { project.commit(id) } + Gitlab::GitalyClient.allow_n_plus_1_calls("gitlab-ce#43894") { project.commit(id) } end end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 80c84c0f622..34ae3cbb72e 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -153,7 +153,7 @@ module Banzai def uri_type(path) # https://gitlab.com/gitlab-org/gitlab-ce/issues/58657 - Gitlab::GitalyClient.allow_n_plus_1_calls do + Gitlab::GitalyClient.allow_n_plus_1_calls("gitlab-ce#58657") do @uri_types[path] ||= current_commit.uri_type(path) end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index ea0d8c85a66..19779f2993d 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -62,7 +62,7 @@ module Gitlab def process_commits logger.log_timed(LOG_MESSAGES[:diff_content_check]) do # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do + ::Gitlab::GitalyClient.allow_n_plus_1_calls("gitlab-ee#3593") do commits.each do |commit| logger.check_timeout_reached diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 40bda3410e1..d41abec76a8 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -73,7 +73,7 @@ module Gitlab # For performance purposes maximum 20 latest commits # will be passed as post receive hook data. # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38259 - commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls do + commit_attrs = Gitlab::GitalyClient.allow_n_plus_1_calls("gitlab-ce#38259") do commits_limited.map do |commit| commit.hook_attrs(with_changed_files: true) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e4d4779ba9a..fbdf9353d57 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,6 +2,8 @@ module Gitlab module Database + include Gitlab::Metrics::Methods + # The max value of INTEGER type is the same between MySQL and PostgreSQL: # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html @@ -11,6 +13,10 @@ module Gitlab # https://dev.mysql.com/doc/refman/5.7/en/datetime.html MAX_TIMESTAMP_VALUE = Time.at((1 << 31) - 1).freeze + define_histogram :gitlab_database_transaction_seconds do + docstring "Time spent in database transactions, in seconds" + end + def self.config ActiveRecord::Base.configurations[Rails.env] end @@ -286,5 +292,31 @@ module Gitlab 0 end private_class_method :open_transactions_baseline + + # Monkeypatch rails with upgraded database observability + def self.install_monkey_patches + ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) + end + + # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to + # record transaction durations. + def self.observe_transaction_duration(duration_seconds) + labels = Gitlab::Metrics::Transaction.current&.labels || {} + gitlab_database_transaction_seconds.observe(labels, duration_seconds) + rescue + # Ensure that errors in recording these metrics don't affect the operation of the application + end + + # MonkeyPatch for ActiveRecord::Base for adding observability + module ActiveRecordBaseTransactionMetrics + # A monkeypatch over ActiveRecord::Base.transaction. + # It provides observability into transactional methods. + def transaction(options = {}, &block) + start_time = Gitlab::Metrics::System.monotonic_time + self.method(:transaction).super_method.call(options, &block) + ensure + Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time) + end + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 47976389af6..c4c91fbb538 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -38,6 +38,14 @@ module Gitlab base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) end + define_histogram :n_plus_one_histogram do + docstring "Time spent in Gitaly n plus one blocks, in seconds" + end + + define_counter :n_plus_one_call_counter do + docstring "Calls made in Gitaly n plus one blocks" + end + def self.stub(name, storage) MUTEX.synchronize do @stubs ||= {} @@ -285,15 +293,18 @@ module Gitlab end private_class_method :enforce_gitaly_request_limits? - def self.allow_n_plus_1_calls - return yield unless Gitlab::SafeRequestStore.active? + def self.allow_n_plus_1_calls(call_site) + increment_call_count(:gitaly_call_count_exception_block_depth) if Gitlab::SafeRequestStore.active? + start_time = Gitlab::Metrics::System.monotonic_time + start_call_count = get_request_count + yield + ensure + labels = current_transaction_labels.merge(transaction: Gitlab::Database.inside_transaction? ? 1 : 0, call_site: call_site) - begin - increment_call_count(:gitaly_call_count_exception_block_depth) - yield - ensure - decrement_call_count(:gitaly_call_count_exception_block_depth) - end + n_plus_one_histogram.observe(labels, Gitlab::Metrics::System.monotonic_time - start_time) + n_plus_one_call_counter.increment(labels, get_request_count - start_call_count) + + decrement_call_count(:gitaly_call_count_exception_block_depth) if Gitlab::SafeRequestStore.active? end # Normally a FindCommit RPC will cache the commit with its SHA diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 868b2ae641a..0c0f46d3b77 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -5,6 +5,7 @@ module Gitlab module_function def retry_lock(subject, retries = 100, &block) + # TODO(Observability): We should be recording details of the number of retries and the duration of the total execution here ActiveRecord::Base.transaction do yield(subject) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 93182607616..1c4f4676c79 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -262,7 +262,7 @@ module Gitlab # def add_namespace(storage, name) # https://gitlab.com/gitlab-org/gitlab-ce/issues/58012 - Gitlab::GitalyClient.allow_n_plus_1_calls do + Gitlab::GitalyClient.allow_n_plus_1_calls("gitlab-ce#58012") do Gitlab::GitalyClient::NamespaceService.new(storage).add(name) end rescue GRPC::InvalidArgument => e |