diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
tree | 78be5963ec075d80116a932011d695dd33910b4e /lib/gitlab/usage_data_counters | |
parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'lib/gitlab/usage_data_counters')
-rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 149 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/known_events.yml | 88 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/track_unique_actions.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/wiki_page_counter.rb | 2 |
4 files changed, 243 insertions, 20 deletions
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb new file mode 100644 index 00000000000..c9c39225068 --- /dev/null +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module HLLRedisCounter + DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks + DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + DEFAULT_REDIS_SLOT = ''.freeze + + UnknownEvent = Class.new(StandardError) + UnknownAggregation = Class.new(StandardError) + + KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze + ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + + # Track event on entity_id + # Increment a Redis HLL counter for unique event_name and entity_id + # + # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml + # + # Event example: + # + # - name: g_compliance_dashboard # Unique event name + # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals + # category: compliance # Group events in categories + # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly + # aggregation: daily # Aggregation level, keys are stored daily or weekly + # + # Usage: + # + # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') + # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) + class << self + def track_event(entity_id, event_name, time = Time.zone.now) + event = event_for(event_name) + + raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? + + Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event)) + end + + def unique_events(event_names:, start_date:, end_date:) + events = events_for(Array(event_names)) + + raise 'Events should be in same slot' unless events_in_same_slot?(events) + raise 'Events should be in same category' unless events_in_same_category?(events) + raise 'Events should have same aggregation level' unless events_same_aggregation?(events) + + aggregation = events.first[:aggregation] + + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) + + Gitlab::Redis::HLL.count(keys: keys) + end + + def events_for_category(category) + known_events.select { |event| event[:category] == category }.map { |event| event[:name] } + end + + private + + def keys_for_aggregation(aggregation, events:, start_date:, end_date:) + if aggregation.to_sym == :daily + daily_redis_keys(events: events, start_date: start_date, end_date: end_date) + else + weekly_redis_keys(events: events, start_date: start_date, end_date: end_date) + end + end + + def known_events + @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access) + end + + def known_events_names + known_events.map { |event| event[:name] } + end + + def events_in_same_slot?(events) + slot = events.first[:redis_slot] + events.all? { |event| event[:redis_slot] == slot } + end + + def events_in_same_category?(events) + category = events.first[:category] + events.all? { |event| event[:category] == category } + end + + def events_same_aggregation?(events) + aggregation = events.first[:aggregation] + events.all? { |event| event[:aggregation] == aggregation } + end + + def expiry(event) + return event[:expiry] if event[:expiry].present? + + event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH + end + + def event_for(event_name) + known_events.find { |event| event[:name] == event_name } + end + + def events_for(event_names) + known_events.select { |event| event_names.include?(event[:name]) } + end + + def redis_slot(event) + event[:redis_slot] || DEFAULT_REDIS_SLOT + end + + # Compose the key in order to store events daily or weekly + def redis_key(event, time) + raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) + raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + + slot = redis_slot(event) + key = if slot.present? + event[:name].to_s.gsub(slot, "{#{slot}}") + else + "{#{event[:name]}}" + end + + if event[:aggregation].to_sym == :daily + year_day = time.strftime('%G-%j') + "#{year_day}-#{key}" + else + year_week = time.strftime('%G-%V') + "#{key}-#{year_week}" + end + end + + def daily_redis_keys(events:, start_date:, end_date:) + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date) } + end.flatten + end + + def weekly_redis_keys(events:, start_date:, end_date:) + weeks = end_date.to_date.cweek - start_date.to_date.cweek + weeks = 1 if weeks == 0 + + (0..(weeks - 1)).map do |week_increment| + events.map { |event| redis_key(event, start_date + week_increment * 7.days) } + end.flatten + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml new file mode 100644 index 00000000000..b7e516fa8b1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -0,0 +1,88 @@ +--- +# Compliance category +- name: g_compliance_dashboard + redis_slot: compliance + category: compliance + expiry: 84 # expiration time in days, equivalent to 12 weeks + aggregation: weekly +- name: g_compliance_audit_events + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +- name: i_compliance_audit_events + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +- name: i_compliance_credential_inventory + category: compliance + redis_slot: compliance + expiry: 84 + aggregation: weekly +# Analytics category +- name: g_analytics_contribution + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_insights + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_issues + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_productivity + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: g_analytics_valuestream + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_pipelines + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_code_reviews + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_valuestream + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_insights + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_issues + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: p_analytics_repo + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: i_analytics_cohorts + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly +- name: i_analytics_dev_ops_score + category: analytics + redis_slot: analytics + expiry: 84 + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_actions.rb index 9fb5a29748e..0df982572a4 100644 --- a/lib/gitlab/usage_data_counters/track_unique_actions.rb +++ b/lib/gitlab/usage_data_counters/track_unique_actions.rb @@ -4,7 +4,6 @@ module Gitlab module UsageDataCounters module TrackUniqueActions KEY_EXPIRY_LENGTH = 29.days - FEATURE_FLAG = :track_unique_actions WIKI_ACTION = :wiki_action DESIGN_ACTION = :design_action @@ -27,24 +26,22 @@ module Gitlab }).freeze class << self - def track_action(event_action:, event_target:, author_id:, time: Time.zone.now) + def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) return unless Gitlab::CurrentSettings.usage_ping_enabled - return unless Feature.enabled?(FEATURE_FLAG) return unless valid_target?(event_target) return unless valid_action?(event_action) transformed_target = transform_target(event_target) transformed_action = transform_action(event_action, transformed_target) + target_key = key(transformed_action, time) - add_event(transformed_action, author_id, time) + Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH) end - def count_unique_events(event_action:, date_from:, date_to:) + def count_unique(event_action:, date_from:, date_to:) keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) } - Gitlab::Redis::SharedState.with do |redis| - redis.pfcount(*keys) - end + Gitlab::Redis::HLL.count(keys: keys) end private @@ -69,17 +66,6 @@ module Gitlab year_day = date.strftime('%G-%j') "#{year_day}-{#{event_action}}" end - - def add_event(event_action, author_id, date) - target_key = key(event_action, date) - - Gitlab::Redis::SharedState.with do |redis| - redis.multi do |multi| - multi.pfadd(target_key, author_id) - multi.expire(target_key, KEY_EXPIRY_LENGTH) - end - end - end end end end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb index 9cfe0be5bab..6c3fe842344 100644 --- a/lib/gitlab/usage_data_counters/wiki_page_counter.rb +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -2,7 +2,7 @@ module Gitlab::UsageDataCounters class WikiPageCounter < BaseCounter - KNOWN_EVENTS = %w[create update delete].freeze + KNOWN_EVENTS = %w[view create update delete].freeze PREFIX = 'wiki_pages' end end |