diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
commit | 85dc423f7090da0a52c73eb66faf22ddb20efff9 (patch) | |
tree | 9160f299afd8c80c038f08e1545be119f5e3f1e1 /lib/gitlab/usage_data_counters | |
parent | 15c2c8c66dbe422588e5411eee7e68f1fa440bb8 (diff) |
Add latest changes from gitlab-org/gitlab@13-4-stable-ee
Diffstat (limited to 'lib/gitlab/usage_data_counters')
-rw-r--r-- | lib/gitlab/usage_data_counters/editor_unique_counter.rb | 56 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 51 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb | 45 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/known_events.yml | 154 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/redis_counter.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/track_unique_events.rb (renamed from lib/gitlab/usage_data_counters/track_unique_actions.rb) | 28 |
7 files changed, 326 insertions, 36 deletions
diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb new file mode 100644 index 00000000000..b68d50ee419 --- /dev/null +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module EditorUniqueCounter + EDIT_BY_SNIPPET_EDITOR = 'g_edit_by_snippet_ide' + EDIT_BY_SFE = 'g_edit_by_sfe' + EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' + EDIT_CATEGORY = 'ide_edit' + + class << self + def track_web_ide_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_WEB_IDE, author, time) + end + + def count_web_ide_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_WEB_IDE, date_from, date_to) + end + + def track_sfe_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_SFE, author, time) + end + + def count_sfe_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_SFE, date_from, date_to) + end + + def track_snippet_editor_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time) + end + + def count_snippet_editor_edit_actions(date_from:, date_to:) + count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to) + end + + def count_edit_using_editor(date_from:, date_to:) + events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(EDIT_CATEGORY) + count_unique(events, date_from, date_to) + end + + private + + def track_unique_action(action, author, time) + return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + end + + def count_unique(actions, date_from, date_to) + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: actions, start_date: date_from, end_date: date_to) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index c9c39225068..53bf6daea4c 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -31,7 +31,11 @@ module Gitlab # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self + include Gitlab::Utils::UsageData + def track_event(entity_id, event_name, time = Time.zone.now) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + event = event_for(event_name) raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? @@ -50,15 +54,51 @@ module Gitlab keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) - Gitlab::Redis::HLL.count(keys: keys) + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + end + + def categories + @categories ||= known_events.map { |event| event[:category] }.uniq end + # @param category [String] the category name + # @return [Array<String>] list of event names for given category def events_for_category(category) - known_events.select { |event| event[:category] == category }.map { |event| event[:name] } + known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } + end + + def unique_events_data + categories.each_with_object({}) do |category, category_results| + events_names = events_for_category(category) + + event_results = events_names.each_with_object({}) do |event, hash| + hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + end + + if eligible_for_totals?(events_names) + event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current) + event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current) + end + + category_results["#{category}"] = event_results + end + end + + def known_event?(event_name) + event_for(event_name).present? end private + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level + # and if there are more than 1 event + def eligible_for_totals?(events_names) + return false if events_names.size <= 1 + + events = events_for(events_names) + events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) + end + def keys_for_aggregation(aggregation, events:, start_date:, end_date:) if aggregation.to_sym == :daily daily_redis_keys(events: events, start_date: start_date, end_date: end_date) @@ -76,8 +116,11 @@ module Gitlab end def events_in_same_slot?(events) + # if we check one event then redis_slot is only one to check + return true if events.size == 1 + slot = events.first[:redis_slot] - events.all? { |event| event[:redis_slot] == slot } + events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot } end def events_in_same_category?(events) @@ -91,7 +134,7 @@ module Gitlab end def expiry(event) - return event[:expiry] if event[:expiry].present? + return event[:expiry].days if event[:expiry].present? event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb new file mode 100644 index 00000000000..fc1b5a59487 --- /dev/null +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module IssueActivityUniqueCounter + ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' + ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed' + ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' + ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' + ISSUE_CATEGORY = 'issues_edit' + + class << self + def track_issue_title_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_TITLE_CHANGED, author, time) + end + + def track_issue_description_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time) + end + + def track_issue_assignee_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time) + end + + def track_issue_made_confidential_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time) + end + + def track_issue_made_visible_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MADE_VISIBLE, author, time) + end + + private + + def track_unique_action(action, author, time) + return unless Feature.enabled?(:track_issue_activity_actions) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index b7e516fa8b1..25e7f858bb1 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -3,86 +3,206 @@ - name: g_compliance_dashboard redis_slot: compliance category: compliance - expiry: 84 # expiration time in days, equivalent to 12 weeks aggregation: weekly - name: g_compliance_audit_events category: compliance redis_slot: compliance - expiry: 84 aggregation: weekly - name: i_compliance_audit_events category: compliance redis_slot: compliance - expiry: 84 aggregation: weekly - name: i_compliance_credential_inventory category: compliance redis_slot: compliance - expiry: 84 + aggregation: weekly +- name: a_compliance_audit_events_api + category: compliance + redis_slot: compliance aggregation: weekly # Analytics category - name: g_analytics_contribution category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_insights category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_issues category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_productivity category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: g_analytics_valuestream category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_pipelines category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_code_reviews category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_valuestream category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_insights category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_issues category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: p_analytics_repo category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: i_analytics_cohorts category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics - expiry: 84 aggregation: weekly +- name: g_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly +- name: i_analytics_instance_statistics + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_edit_by_web_ide + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: g_edit_by_sfe + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: g_edit_by_snippet_ide + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily +- name: i_search_total + category: search + redis_slot: search + aggregation: weekly +- name: i_search_advanced + category: search + redis_slot: search + aggregation: weekly +- name: i_search_paid + category: search + redis_slot: search + aggregation: weekly +- name: wiki_action + category: source_code + aggregation: daily +- name: design_action + category: source_code + aggregation: daily +- name: project_action + category: source_code + aggregation: daily +- name: merge_request_action + category: source_code + aggregation: daily +- name: i_source_code_code_intelligence + redis_slot: source_code + category: source_code + aggregation: daily +# Incident management +- name: incident_management_alert_status_changed + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_alert_assigned + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_alert_todo + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_created + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_reopened + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_closed + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_assigned + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_todo + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_comment + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_zoom_meeting + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_published + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_relate + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_unrelate + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_incident_change_confidential + redis_slot: incident_management + category: incident_management + aggregation: weekly +# Project Management group +- name: g_project_management_issue_title_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_description_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_assignee_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_made_confidential + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_made_visible + category: issues_edit + redis_slot: project_management + aggregation: daily diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb new file mode 100644 index 00000000000..eae42bdc4a1 --- /dev/null +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class KubernetesAgentCounter < BaseCounter + PREFIX = 'kubernetes_agent' + KNOWN_EVENTS = %w[gitops_sync].freeze + + class << self + def increment_gitops_sync(incr) + raise ArgumentError, 'must be greater than or equal to zero' if incr < 0 + + # rather then hitting redis for this no-op, we return early + # note: redis returns the increment, so we mimic this here + return 0 if incr == 0 + + increment_by(redis_key(:gitops_sync), incr) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 75d5a75e3a4..2406f771fd8 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -9,6 +9,12 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end + def increment_by(redis_counter_key, incr) + return unless Gitlab::CurrentSettings.usage_ping_enabled + + Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) } + end + def total_count(redis_counter_key) Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i } end diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 0df982572a4..7053744b665 100644 --- a/lib/gitlab/usage_data_counters/track_unique_actions.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -2,12 +2,11 @@ module Gitlab module UsageDataCounters - module TrackUniqueActions - KEY_EXPIRY_LENGTH = 29.days - + module TrackUniqueEvents WIKI_ACTION = :wiki_action DESIGN_ACTION = :design_action PUSH_ACTION = :project_action + MERGE_REQUEST_ACTION = :merge_request_action ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { @@ -22,26 +21,30 @@ module Gitlab }, project: { pushed: PUSH_ACTION + }, + merge_request: { + closed: MERGE_REQUEST_ACTION, + merged: MERGE_REQUEST_ACTION, + created: MERGE_REQUEST_ACTION, + commented: MERGE_REQUEST_ACTION } }).freeze class << self def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled return unless valid_target?(event_target) return unless valid_action?(event_action) transformed_target = transform_target(event_target) transformed_action = transform_action(event_action, transformed_target) - target_key = key(transformed_action, time) - Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH) - end + return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - def count_unique(event_action:, date_from:, date_to:) - keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) } + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + end - Gitlab::Redis::HLL.count(keys: keys) + def count_unique_events(event_action:, date_from:, date_to:) + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to) end private @@ -61,11 +64,6 @@ module Gitlab def valid_action?(action) Event.actions.key?(action) end - - def key(event_action, date) - year_day = date.strftime('%G-%j') - "#{year_day}-{#{event_action}}" - end end end end |