diff options
Diffstat (limited to 'lib/gitlab/usage_data_counters/hll_redis_counter.rb')
| -rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 80 |
1 files changed, 14 insertions, 66 deletions
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index badcda1def0..eaa4bf15fe1 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -3,18 +3,14 @@ module Gitlab module UsageDataCounters module HLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + KEY_EXPIRY_LENGTH = 6.weeks REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) - UnknownAggregation = Class.new(EventError) - AggregationMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i(daily weekly).freeze # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -24,7 +20,6 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # aggregation: weekly # Aggregation level, keys are stored weekly # # Usage: # @@ -63,8 +58,7 @@ module Gitlab # end_date - The end date of the time range. # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do raise InvalidContext if context.present? && !context.in?(valid_context_list) end end @@ -78,9 +72,7 @@ module Gitlab end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) - end + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) end private @@ -94,12 +86,7 @@ module Gitlab return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - if event[:aggregation].to_sym == :daily - weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' } - Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event)) - end - - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -117,25 +104,18 @@ module Gitlab yield events if block_given? - aggregation = events.first[:aggregation] - - if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation) - aggregation = 'weekly' - events = events.map { |e| e.merge(aggregation: 'weekly') } - end + keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) return FALLBACK unless keys.any? redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') - if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - end + def keys_for_aggregation(events:, start_date:, end_date:, context: '') + end_date = end_date.end_of_week - 1.week + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date, context) } + end.flatten.uniq end def load_events(wildcard) @@ -152,15 +132,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_same_aggregation?(events) - aggregation = events.first[:aggregation] - events.all? { |event| event[:aggregation] == aggregation } - end - - def expiry(event) - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - end - def event_for(event_name) known_events.find { |event| event[:name] == event_name.to_s } end @@ -173,36 +144,13 @@ module Gitlab def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) - # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup - raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end + year_week = time.strftime('%G-%V') + key = "#{key}-#{year_week}" - def daily_redis_keys(events:, start_date:, end_date:, context: '') - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten - end - - def weekly_redis_keys(events:, start_date:, end_date:, context: '') - end_date = end_date.end_of_week - 1.week - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten.uniq + key = "#{context}_#{key}" if context.present? + key end end end |