diff options
Diffstat (limited to 'lib/gitlab/github_import/object_counter.rb')
-rw-r--r-- | lib/gitlab/github_import/object_counter.rb | 84 |
1 files changed, 84 insertions, 0 deletions
diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb new file mode 100644 index 00000000000..e4835504c2d --- /dev/null +++ b/lib/gitlab/github_import/object_counter.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Count objects fetched or imported from Github. +module Gitlab + module GithubImport + class ObjectCounter + OPERATIONS = %w[fetched imported].freeze + PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' + PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + + GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' + GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' + + CACHING = Gitlab::Cache::Import::Caching + + class << self + def increment(project, object_type, operation) + validate_operation!(operation) + + increment_project_counter(project, object_type, operation) + increment_global_counter(object_type, operation) + end + + def summary(project) + OPERATIONS.each_with_object({}) do |operation, result| + result[operation] = {} + + CACHING + .values_from_set(counter_list_key(project, operation)) + .sort + .each do |counter| + object_type = counter.split('/').last + result[operation][object_type] = CACHING.read_integer(counter) + end + end + end + + private + + # Global counters are long lived, in Prometheus, + # and it's used to report the health of the Github Importer + # in the Grafana Dashboard + # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 + def increment_global_counter(object_type, operation) + key = GLOBAL_COUNTER_KEY % { + operation: operation, + object_type: object_type + } + description = GLOBAL_COUNTER_DESCRIPTION % { + operation: operation, + object_type: object_type.to_s.humanize + } + + Gitlab::Metrics.counter(key.to_sym, description).increment + end + + # Project counters are short lived, in Redis, + # and it's used to report how successful a project + # import was with the #summary method. + def increment_project_counter(project, object_type, operation) + counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + + add_counter_to_list(project, operation, counter_key) + + CACHING.increment(counter_key) + end + + def add_counter_to_list(project, operation, key) + CACHING.set_add(counter_list_key(project, operation), key) + end + + def counter_list_key(project, operation) + PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation } + end + + def validate_operation!(operation) + unless operation.to_s.presence_in(OPERATIONS) + raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}" + end + end + end + end + end +end |