Add latest changes from gitlab-org/gitlab@13-4-stable-ee

7 files changed, 326 insertions, 36 deletions

diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb
new file mode 100644
index 00000000000..b68d50ee419
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    module EditorUniqueCounter
+      EDIT_BY_SNIPPET_EDITOR = 'g_edit_by_snippet_ide'
+      EDIT_BY_SFE = 'g_edit_by_sfe'
+      EDIT_BY_WEB_IDE = 'g_edit_by_web_ide'
+      EDIT_CATEGORY = 'ide_edit'
+
+      class << self
+        def track_web_ide_edit_action(author:, time: Time.zone.now)
+          track_unique_action(EDIT_BY_WEB_IDE, author, time)
+        end
+
+        def count_web_ide_edit_actions(date_from:, date_to:)
+          count_unique(EDIT_BY_WEB_IDE, date_from, date_to)
+        end
+
+        def track_sfe_edit_action(author:, time: Time.zone.now)
+          track_unique_action(EDIT_BY_SFE, author, time)
+        end
+
+        def count_sfe_edit_actions(date_from:, date_to:)
+          count_unique(EDIT_BY_SFE, date_from, date_to)
+        end
+
+        def track_snippet_editor_edit_action(author:, time: Time.zone.now)
+          track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time)
+        end
+
+        def count_snippet_editor_edit_actions(date_from:, date_to:)
+          count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to)
+        end
+
+        def count_edit_using_editor(date_from:, date_to:)
+          events = Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(EDIT_CATEGORY)
+          count_unique(events, date_from, date_to)
+        end
+
+        private
+
+        def track_unique_action(action, author, time)
+          return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true)
+          return unless author
+
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time)
+        end
+
+        def count_unique(actions, date_from, date_to)
+          Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: actions, start_date: date_from, end_date: date_to)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index c9c39225068..53bf6daea4c 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -31,7 +31,11 @@ module Gitlab
       # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard')
       # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
       class << self
+        include Gitlab::Utils::UsageData
+
         def track_event(entity_id, event_name, time = Time.zone.now)
+          return unless Gitlab::CurrentSettings.usage_ping_enabled?
+
           event = event_for(event_name)
 
           raise UnknownEvent.new("Unknown event #{event_name}") unless event.present?
@@ -50,15 +54,51 @@ module Gitlab
 
           keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date)
 
-          Gitlab::Redis::HLL.count(keys: keys)
+          redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
+        end
+
+        def categories
+          @categories ||= known_events.map { |event| event[:category] }.uniq
         end
 
+        # @param category [String] the category name
+        # @return [Array<String>] list of event names for given category
         def events_for_category(category)
-          known_events.select { |event| event[:category] == category }.map { |event| event[:name] }
+          known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] }
+        end
+
+        def unique_events_data
+          categories.each_with_object({}) do |category, category_results|
+            events_names = events_for_category(category)
+
+            event_results = events_names.each_with_object({}) do |event, hash|
+              hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current)
+            end
+
+            if eligible_for_totals?(events_names)
+              event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current)
+              event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current)
+            end
+
+            category_results["#{category}"] = event_results
+          end
+        end
+
+        def known_event?(event_name)
+          event_for(event_name).present?
         end
 
         private
 
+        # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
+        # and if there are more than 1 event
+        def eligible_for_totals?(events_names)
+          return false if events_names.size <= 1
+
+          events = events_for(events_names)
+          events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events)
+        end
+
         def keys_for_aggregation(aggregation, events:, start_date:, end_date:)
           if aggregation.to_sym == :daily
             daily_redis_keys(events: events, start_date: start_date, end_date: end_date)
@@ -76,8 +116,11 @@ module Gitlab
         end
 
         def events_in_same_slot?(events)
+          # if we check one event then redis_slot is only one to check
+          return true if events.size == 1
+
           slot = events.first[:redis_slot]
-          events.all? { |event| event[:redis_slot] == slot }
+          events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot }
         end
 
         def events_in_same_category?(events)
@@ -91,7 +134,7 @@ module Gitlab
         end
 
         def expiry(event)
-          return event[:expiry] if event[:expiry].present?
+          return event[:expiry].days if event[:expiry].present?
 
           event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
         end
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
new file mode 100644
index 00000000000..fc1b5a59487
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    module IssueActivityUniqueCounter
+      ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed'
+      ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed'
+      ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed'
+      ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential'
+      ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible'
+      ISSUE_CATEGORY = 'issues_edit'
+
+      class << self
+        def track_issue_title_changed_action(author:, time: Time.zone.now)
+          track_unique_action(ISSUE_TITLE_CHANGED, author, time)
+        end
+
+        def track_issue_description_changed_action(author:, time: Time.zone.now)
+          track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time)
+        end
+
+        def track_issue_assignee_changed_action(author:, time: Time.zone.now)
+          track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time)
+        end
+
+        def track_issue_made_confidential_action(author:, time: Time.zone.now)
+          track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time)
+        end
+
+        def track_issue_made_visible_action(author:, time: Time.zone.now)
+          track_unique_action(ISSUE_MADE_VISIBLE, author, time)
+        end
+
+        private
+
+        def track_unique_action(action, author, time)
+          return unless Feature.enabled?(:track_issue_activity_actions)
+          return unless author
+
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml
index b7e516fa8b1..25e7f858bb1 100644
--- a/lib/gitlab/usage_data_counters/known_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events.yml
@@ -3,86 +3,206 @@
 - name: g_compliance_dashboard
   redis_slot: compliance
   category: compliance
-  expiry: 84 # expiration time in days, equivalent to 12 weeks
   aggregation: weekly
 - name: g_compliance_audit_events
   category: compliance
   redis_slot: compliance
-  expiry: 84
   aggregation: weekly
 - name: i_compliance_audit_events
   category: compliance
   redis_slot: compliance
-  expiry: 84
   aggregation: weekly
 - name: i_compliance_credential_inventory
   category: compliance
   redis_slot: compliance
-  expiry: 84
+  aggregation: weekly
+- name: a_compliance_audit_events_api
+  category: compliance
+  redis_slot: compliance
   aggregation: weekly
 # Analytics category
 - name: g_analytics_contribution
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_insights
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_issues
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_productivity
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: g_analytics_valuestream
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_pipelines
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_code_reviews
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_valuestream
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_insights
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_issues
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: p_analytics_repo
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: i_analytics_cohorts
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
 - name: i_analytics_dev_ops_score
   category: analytics
   redis_slot: analytics
-  expiry: 84
   aggregation: weekly
+- name: g_analytics_merge_request
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+- name: p_analytics_merge_request
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+- name: i_analytics_instance_statistics
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+- name: g_edit_by_web_ide
+  category: ide_edit
+  redis_slot: edit
+  expiry: 29
+  aggregation: daily
+- name: g_edit_by_sfe
+  category: ide_edit
+  redis_slot: edit
+  expiry: 29
+  aggregation: daily
+- name: g_edit_by_snippet_ide
+  category: ide_edit
+  redis_slot: edit
+  expiry: 29
+  aggregation: daily
+- name: i_search_total
+  category: search
+  redis_slot: search
+  aggregation: weekly
+- name: i_search_advanced
+  category: search
+  redis_slot: search
+  aggregation: weekly
+- name: i_search_paid
+  category: search
+  redis_slot: search
+  aggregation: weekly
+- name: wiki_action
+  category: source_code
+  aggregation: daily
+- name: design_action
+  category: source_code
+  aggregation: daily
+- name: project_action
+  category: source_code
+  aggregation: daily
+- name: merge_request_action
+  category: source_code
+  aggregation: daily
+- name: i_source_code_code_intelligence
+  redis_slot: source_code
+  category: source_code
+  aggregation: daily
+# Incident management
+- name: incident_management_alert_status_changed
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_alert_assigned
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_alert_todo
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_created
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_reopened
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_closed
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_assigned
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_todo
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_comment
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_zoom_meeting
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_published
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_relate
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_unrelate
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+- name: incident_management_incident_change_confidential
+  redis_slot: incident_management
+  category: incident_management
+  aggregation: weekly
+# Project Management group
+- name: g_project_management_issue_title_changed
+  category: issues_edit
+  redis_slot: project_management
+  aggregation: daily
+- name: g_project_management_issue_description_changed
+  category: issues_edit
+  redis_slot: project_management
+  aggregation: daily
+- name: g_project_management_issue_assignee_changed
+  category: issues_edit
+  redis_slot: project_management
+  aggregation: daily
+- name: g_project_management_issue_made_confidential
+  category: issues_edit
+  redis_slot: project_management
+  aggregation: daily
+- name: g_project_management_issue_made_visible
+  category: issues_edit
+  redis_slot: project_management
+  aggregation: daily
diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb
new file mode 100644
index 00000000000..eae42bdc4a1
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module UsageDataCounters
+    class KubernetesAgentCounter < BaseCounter
+      PREFIX = 'kubernetes_agent'
+      KNOWN_EVENTS = %w[gitops_sync].freeze
+
+      class << self
+        def increment_gitops_sync(incr)
+          raise ArgumentError, 'must be greater than or equal to zero' if incr < 0
+
+          # rather then hitting redis for this no-op, we return early
+          # note: redis returns the increment, so we mimic this here
+          return 0 if incr == 0
+
+          increment_by(redis_key(:gitops_sync), incr)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb
index 75d5a75e3a4..2406f771fd8 100644
--- a/lib/gitlab/usage_data_counters/redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/redis_counter.rb
@@ -9,6 +9,12 @@ module Gitlab
         Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
       end
 
+      def increment_by(redis_counter_key, incr)
+        return unless Gitlab::CurrentSettings.usage_ping_enabled
+
+        Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) }
+      end
+
       def total_count(redis_counter_key)
         Gitlab::Redis::SharedState.with { |redis| redis.get(redis_counter_key).to_i }
       end
diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb
index 0df982572a4..7053744b665 100644
--- a/lib/gitlab/usage_data_counters/track_unique_actions.rb
+++ b/lib/gitlab/usage_data_counters/track_unique_events.rb
@@ -2,12 +2,11 @@
 
 module Gitlab
   module UsageDataCounters
-    module TrackUniqueActions
-      KEY_EXPIRY_LENGTH = 29.days
-
+    module TrackUniqueEvents
       WIKI_ACTION = :wiki_action
       DESIGN_ACTION = :design_action
       PUSH_ACTION = :project_action
+      MERGE_REQUEST_ACTION = :merge_request_action
 
       ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({
         wiki: {
@@ -22,26 +21,30 @@ module Gitlab
         },
         project: {
           pushed: PUSH_ACTION
+        },
+        merge_request: {
+          closed: MERGE_REQUEST_ACTION,
+          merged: MERGE_REQUEST_ACTION,
+          created: MERGE_REQUEST_ACTION,
+          commented: MERGE_REQUEST_ACTION
         }
       }).freeze
 
       class << self
         def track_event(event_action:, event_target:, author_id:, time: Time.zone.now)
-          return unless Gitlab::CurrentSettings.usage_ping_enabled
           return unless valid_target?(event_target)
           return unless valid_action?(event_action)
 
           transformed_target = transform_target(event_target)
           transformed_action = transform_action(event_action, transformed_target)
-          target_key = key(transformed_action, time)
 
-          Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH)
-        end
+          return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s)
 
-        def count_unique(event_action:, date_from:, date_to:)
-          keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) }
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time)
+        end
 
-          Gitlab::Redis::HLL.count(keys: keys)
+        def count_unique_events(event_action:, date_from:, date_to:)
+          Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to)
         end
 
         private
@@ -61,11 +64,6 @@ module Gitlab
         def valid_action?(action)
           Event.actions.key?(action)
         end
-
-        def key(event_action, date)
-          year_day = date.strftime('%G-%j')
-          "#{year_day}-{#{event_action}}"
-        end
       end
     end
   end