diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/commits.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/instrumentation/redis_helper.rb | 26 | ||||
| -rw-r--r-- | lib/gitlab/patch/redis_client.rb | 18 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/editor_unique_counter.rb | 55 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 56 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml | 128 |
6 files changed, 208 insertions, 78 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 021b3a9437c..7570702444d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -32,7 +32,8 @@ module API return unless find_user_from_warden Gitlab::UsageDataCounters::WebIdeCounter.increment_commits_count - Gitlab::UsageDataCounters::EditorUniqueCounter.track_web_ide_edit_action(author: current_user, project: user_project) + Gitlab::InternalEvents.track_event('g_edit_by_web_ide', user: current_user, project: user_project) + namespace = user_project.namespace Gitlab::Tracking.event( diff --git a/lib/gitlab/instrumentation/redis_helper.rb b/lib/gitlab/instrumentation/redis_helper.rb index 392a7ebe852..8061c3702d5 100644 --- a/lib/gitlab/instrumentation/redis_helper.rb +++ b/lib/gitlab/instrumentation/redis_helper.rb @@ -16,13 +16,17 @@ module Gitlab yield rescue ::Redis::BaseError, ::RedisClient::Error => ex - if ex.message.start_with?('MOVED', 'ASK') - instrumentation_class.instance_count_cluster_redirection(ex) - else - instrumentation_class.instance_count_exception(ex) + Thread.current[:redis_client_error_count] ||= 0 + + # skip instrumentation if the error is a connection error happening for the first time as instrumentation + # middlewares are called within `ensure_connected` blocks. Connection retries are not known to the middleware. + # Refer to https://github.com/redis-rb/redis-client/issues/119#issuecomment-1829703792 + unless ex.is_a?(::RedisClient::ConnectionError) && Thread.current[:redis_client_error_count] == 0 + instrument_errors(instrumentation_class, ex) end - instrumentation_class.log_exception(ex) + Thread.current[:redis_client_error_count] += 1 if ex.is_a?(::RedisClient::Error) + raise ex ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -80,6 +84,18 @@ module Gitlab def exclude_from_apdex?(commands) commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } end + + private + + def instrument_errors(instrumentation_class, error) + if error.message.start_with?('MOVED', 'ASK') + instrumentation_class.instance_count_cluster_redirection(error) + else + instrumentation_class.instance_count_exception(error) + end + + instrumentation_class.log_exception(error) + end end end end diff --git a/lib/gitlab/patch/redis_client.rb b/lib/gitlab/patch/redis_client.rb new file mode 100644 index 00000000000..24fa4cf34b2 --- /dev/null +++ b/lib/gitlab/patch/redis_client.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module RedisClient + # This patch resets the connection error tracker after each call to prevent state leak + # across calls and requests. + # + # The purpose of the tracker is to silence RedisClient::ConnectionErrors during reconnection attempts. + # More details found in https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2564#note_1665334335 + def ensure_connected(retryable: true) + super + ensure + Thread.current[:redis_client_error_count] = 0 + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb deleted file mode 100644 index 7955c19b7e6..00000000000 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - module EditorUniqueCounter - EDIT_BY_SNIPPET_EDITOR = 'g_edit_by_snippet_ide' - EDIT_BY_SFE = 'g_edit_by_sfe' - EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' - EDIT_CATEGORY = 'ide_edit' - - class << self - def track_web_ide_edit_action(author:, project:) - track_internal_event(EDIT_BY_WEB_IDE, author, project) - end - - def count_web_ide_edit_actions(date_from:, date_to:) - count_unique(EDIT_BY_WEB_IDE, date_from, date_to) - end - - def track_sfe_edit_action(author:, project:) - track_internal_event(EDIT_BY_SFE, author, project) - end - - def count_sfe_edit_actions(date_from:, date_to:) - count_unique(EDIT_BY_SFE, date_from, date_to) - end - - def track_snippet_editor_edit_action(author:, project:) - track_internal_event(EDIT_BY_SNIPPET_EDITOR, author, project) - end - - def count_snippet_editor_edit_actions(date_from:, date_to:) - count_unique(EDIT_BY_SNIPPET_EDITOR, date_from, date_to) - end - - private - - def track_internal_event(event_name, author, project = nil) - return unless author - - Gitlab::InternalEvents.track_event( - event_name, - user: author, - project: project, - namespace: project&.namespace - ) - end - - def count_unique(actions, date_from, date_to) - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: actions, start_date: date_from, end_date: date_to) - end - end - end - end -end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 137b6f90545..97039a58e92 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -29,18 +29,20 @@ module Gitlab # # event_name - The event name. # values - One or multiple values counted. + # property_name - Name of the values counted. # time - Time of the action, set to Time.current. - def track_event(event_name, values:, time: Time.current) - track(values, event_name, time: time) + def track_event(event_name, values:, property_name: nil, time: Time.current) + track(values, event_name, property_name: property_name, time: time) end # Count unique events for a given time range. # # event_names - The list of the events to count. + # property_names - The list of the values for which the events are to be counted. # start_date - The start date of the time range. # end_date - The end date of the time range. - def unique_events(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) + def unique_events(event_names:, start_date:, end_date:, property_name: nil) + count_unique_events(event_names: event_names, property_name: property_name, start_date: start_date, end_date: end_date) end def known_event?(event_name) @@ -52,19 +54,19 @@ module Gitlab end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) + count_unique_events(event_names: event_names, property_name: nil, start_date: start_date, end_date: end_date) end private - def track(values, event_name, time: Time.zone.now) + def track(values, event_name, property_name:, time: Time.zone.now) event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - Gitlab::Redis::HLL.add(key: redis_key(event, time), value: values, expiry: KEY_EXPIRY_LENGTH) + Gitlab::Redis::HLL.add(key: redis_key(event_with_property_name(event, property_name), time), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -72,8 +74,8 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - def count_unique_events(event_names:, start_date:, end_date:) - events = events_for(Array(event_names).map(&:to_s)) + def count_unique_events(event_names:, start_date:, end_date:, property_name:) + events = events_with_property_names(event_names, property_name) keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date) @@ -82,6 +84,19 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end + def events_with_property_names(event_names, property_name) + event_names = Array(event_names).map(&:to_s) + known_events.filter_map do |event| + next unless event_names.include?(event[:name]) + + property_name ? event_with_property_name(event, property_name) : event + end + end + + def event_with_property_name(event, property_name) + event.merge(property_name: property_name) + end + def load_events events = Gitlab::Usage::MetricDefinition.all.map do |d| next unless d.available? @@ -102,21 +117,30 @@ module Gitlab known_events.find { |event| event[:name] == event_name.to_s } end - def events_for(event_names) - known_events.select { |event| event_names.include?(event[:name]) } - end - def redis_key(event, time) - key = redis_key_base(event[:name]) + key = redis_key_base(event) year_week = time.strftime('%G-%V') "{#{REDIS_SLOT}}_#{key}-#{year_week}" end - def redis_key_base(event_name) + def redis_key_base(event) + event_name = event[:name] + raise UnknownEvent, "Unknown event #{event_name}" unless known_events_names.include?(event_name.to_s) - key_overrides.fetch(event_name, event_name) + property_name = event[:property_name] + key = event_name + if Feature.enabled?(:redis_hll_property_name_tracking, type: :wip) && property_name + key = "#{key}-#{formatted_property_name(property_name)}" + end + + key_overrides.fetch(key, key) + end + + def formatted_property_name(property_name) + # simplify to format from EventDefinitions.unique_properties + property_name.to_s.split('.').first.to_sym end def key_overrides diff --git a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml index 0967ef424bc..9adbb02ff2c 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml +++ b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml @@ -1 +1,127 @@ -{} +# This file lists all of the internal events that need to be saved with their legacy HLL Redis keys +# +# This file has been generated using the script included in +# the description of https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137890 +# +# It is only safe to regenerate it using the same script if the +# :redis_hll_property_name_tracking feature flag is disabled on prod environment. +--- +agent_users_using_ci_tunnel-user: agent_users_using_ci_tunnel +ci_template_included-project: ci_template_included +exclude_anonymised_users-user: exclude_anonymised_users +g_compliance_dashboard-user: g_compliance_dashboard +g_edit_by_sfe-user: g_edit_by_sfe +g_edit_by_snippet_ide-user: g_edit_by_snippet_ide +g_edit_by_web_ide-user: g_edit_by_web_ide +g_project_management_epic_blocked_added-user: g_project_management_epic_blocked_added +g_project_management_epic_blocked_removed-user: g_project_management_epic_blocked_removed +g_project_management_epic_blocking_added-user: g_project_management_epic_blocking_added +g_project_management_epic_blocking_removed-user: g_project_management_epic_blocking_removed +g_project_management_epic_closed-user: g_project_management_epic_closed +g_project_management_epic_created-user: g_project_management_epic_created +g_project_management_epic_cross_referenced-user: g_project_management_epic_cross_referenced +g_project_management_epic_destroyed-user: g_project_management_epic_destroyed +g_project_management_epic_issue_added-user: g_project_management_epic_issue_added +g_project_management_epic_issue_moved_from_project-user: g_project_management_epic_issue_moved_from_project +g_project_management_epic_issue_removed-user: g_project_management_epic_issue_removed +g_project_management_epic_related_added-user: g_project_management_epic_related_added +g_project_management_epic_related_removed-user: g_project_management_epic_related_removed +g_project_management_epic_reopened-user: g_project_management_epic_reopened +g_project_management_epic_users_changing_labels-user: g_project_management_epic_users_changing_labels +g_project_management_issue_added_to_epic-user: g_project_management_issue_added_to_epic +g_project_management_issue_assignee_changed-user: g_project_management_issue_assignee_changed +g_project_management_issue_changed_epic-user: g_project_management_issue_changed_epic +g_project_management_issue_cloned-user: g_project_management_issue_cloned +g_project_management_issue_closed-user: g_project_management_issue_closed +g_project_management_issue_comment_added-user: g_project_management_issue_comment_added +g_project_management_issue_comment_edited-user: g_project_management_issue_comment_edited +g_project_management_issue_comment_removed-user: g_project_management_issue_comment_removed +g_project_management_issue_created-user: g_project_management_issue_created +g_project_management_issue_cross_referenced-user: g_project_management_issue_cross_referenced +g_project_management_issue_description_changed-user: g_project_management_issue_description_changed +g_project_management_issue_design_comments_removed-user: g_project_management_issue_design_comments_removed +g_project_management_issue_designs_added-user: g_project_management_issue_designs_added +g_project_management_issue_designs_modified-user: g_project_management_issue_designs_modified +g_project_management_issue_designs_removed-user: g_project_management_issue_designs_removed +g_project_management_issue_due_date_changed-user: g_project_management_issue_due_date_changed +g_project_management_issue_health_status_changed-user: g_project_management_issue_health_status_changed +g_project_management_issue_iteration_changed-user: g_project_management_issue_iteration_changed +g_project_management_issue_label_changed-user: g_project_management_issue_label_changed +g_project_management_issue_locked-user: g_project_management_issue_locked +g_project_management_issue_made_confidential-user: g_project_management_issue_made_confidential +g_project_management_issue_made_visible-user: g_project_management_issue_made_visible +g_project_management_issue_marked_as_duplicate-user: g_project_management_issue_marked_as_duplicate +g_project_management_issue_milestone_changed-user: g_project_management_issue_milestone_changed +g_project_management_issue_moved-user: g_project_management_issue_moved +g_project_management_issue_promoted_to_epic-user: g_project_management_issue_promoted_to_epic +g_project_management_issue_related-user: g_project_management_issue_related +g_project_management_issue_removed_from_epic-user: g_project_management_issue_removed_from_epic +g_project_management_issue_reopened-user: g_project_management_issue_reopened +g_project_management_issue_time_estimate_changed-user: g_project_management_issue_time_estimate_changed +g_project_management_issue_time_spent_changed-user: g_project_management_issue_time_spent_changed +g_project_management_issue_title_changed-user: g_project_management_issue_title_changed +g_project_management_issue_unlocked-user: g_project_management_issue_unlocked +g_project_management_issue_unrelated-user: g_project_management_issue_unrelated +g_project_management_issue_weight_changed-user: g_project_management_issue_weight_changed +g_project_management_users_awarding_epic_emoji-user: g_project_management_users_awarding_epic_emoji +g_project_management_users_creating_epic_notes-user: g_project_management_users_creating_epic_notes +g_project_management_users_destroying_epic_notes-user: g_project_management_users_destroying_epic_notes +g_project_management_users_epic_issue_added_from_epic-user: g_project_management_users_epic_issue_added_from_epic +g_project_management_users_removing_epic_emoji-user: g_project_management_users_removing_epic_emoji +g_project_management_users_setting_epic_confidential-user: g_project_management_users_setting_epic_confidential +g_project_management_users_setting_epic_due_date_as_fixed-user: g_project_management_users_setting_epic_due_date_as_fixed +g_project_management_users_setting_epic_due_date_as_inherited-user: g_project_management_users_setting_epic_due_date_as_inherited +g_project_management_users_setting_epic_start_date_as_fixed-user: g_project_management_users_setting_epic_start_date_as_fixed +g_project_management_users_setting_epic_start_date_as_inherited-user: g_project_management_users_setting_epic_start_date_as_inherited +g_project_management_users_setting_epic_visible-user: g_project_management_users_setting_epic_visible +g_project_management_users_updating_epic_descriptions-user: g_project_management_users_updating_epic_descriptions +g_project_management_users_updating_epic_notes-user: g_project_management_users_updating_epic_notes +g_project_management_users_updating_epic_parent-user: g_project_management_users_updating_epic_parent +g_project_management_users_updating_epic_titles-user: g_project_management_users_updating_epic_titles +g_project_management_users_updating_fixed_epic_due_date-user: g_project_management_users_updating_fixed_epic_due_date +g_project_management_users_updating_fixed_epic_start_date-user: g_project_management_users_updating_fixed_epic_start_date +geo_secondary_git_op_action-user: geo_secondary_git_op_action +i_analytics_dev_ops_adoption-user: i_analytics_dev_ops_adoption +i_analytics_dev_ops_score-user: i_analytics_dev_ops_score +i_code_review_saved_replies_create-user: i_code_review_saved_replies_create +i_code_review_saved_replies_use-user: i_code_review_saved_replies_use +i_code_review_saved_replies_use_in_mr-user: i_code_review_saved_replies_use_in_mr +i_code_review_saved_replies_use_in_other-user: i_code_review_saved_replies_use_in_other +i_code_review_user_create_mr-user: i_code_review_user_create_mr +i_quickactions_remove_email_multiple-user: i_quickactions_remove_email_multiple +i_quickactions_remove_email_single-user: i_quickactions_remove_email_single +insights_chart_item_clicked-user: insights_chart_item_clicked +insights_issue_chart_item_clicked-user: insights_issue_chart_item_clicked +k8s_api_proxy_requests_unique_users_via_ci_access-user: k8s_api_proxy_requests_unique_users_via_ci_access +k8s_api_proxy_requests_unique_users_via_pat_access-user: k8s_api_proxy_requests_unique_users_via_pat_access +k8s_api_proxy_requests_unique_users_via_user_access-user: k8s_api_proxy_requests_unique_users_via_user_access +p_analytics_ci_cd_deployment_frequency-user: p_analytics_ci_cd_deployment_frequency +p_analytics_ci_cd_lead_time-user: p_analytics_ci_cd_lead_time +p_analytics_ci_cd_pipelines-user: p_analytics_ci_cd_pipelines +project_management_users_checking_epic_task-user: project_management_users_checking_epic_task +project_management_users_unchecking_epic_task-user: project_management_users_unchecking_epic_task +unique_users_visiting_ci_catalog-user: unique_users_visiting_ci_catalog +user_created_custom_dashboard-user: user_created_custom_dashboard +user_created_custom_visualization-user: user_created_custom_visualization +user_edited_cluster_configuration-user: user_edited_cluster_configuration +user_edited_custom_dashboard-user: user_edited_custom_dashboard +user_viewed_cluster_configuration-user: user_viewed_cluster_configuration +user_viewed_custom_dashboard-user: user_viewed_custom_dashboard +user_viewed_dashboard_designer-user: user_viewed_dashboard_designer +user_viewed_dashboard_list-user: user_viewed_dashboard_list +user_viewed_instrumentation_directions-user: user_viewed_instrumentation_directions +user_viewed_visualization_designer-user: user_viewed_visualization_designer +user_visited_dashboard-user: user_visited_dashboard +value_streams_dashboard_change_failure_rate_link_clicked-user: value_streams_dashboard_change_failure_rate_link_clicked +value_streams_dashboard_cycle_time_link_clicked-user: value_streams_dashboard_cycle_time_link_clicked +value_streams_dashboard_deployment_frequency_link_clicked-user: value_streams_dashboard_deployment_frequency_link_clicked +value_streams_dashboard_deploys_link_clicked-user: value_streams_dashboard_deploys_link_clicked +value_streams_dashboard_issues_completed_link_clicked-user: value_streams_dashboard_issues_completed_link_clicked +value_streams_dashboard_issues_link_clicked-user: value_streams_dashboard_issues_link_clicked +value_streams_dashboard_lead_time_for_changes_link_clicked-user: value_streams_dashboard_lead_time_for_changes_link_clicked +value_streams_dashboard_lead_time_link_clicked-user: value_streams_dashboard_lead_time_link_clicked +value_streams_dashboard_merge_request_throughput_link_clicked-user: value_streams_dashboard_merge_request_throughput_link_clicked +value_streams_dashboard_metric_link_clicked-user: value_streams_dashboard_metric_link_clicked +value_streams_dashboard_time_to_restore_service_link_clicked-user: value_streams_dashboard_time_to_restore_service_link_clicked +value_streams_dashboard_vulnerability_critical_link_clicked-user: value_streams_dashboard_vulnerability_critical_link_clicked +value_streams_dashboard_vulnerability_high_link_clicked-user: value_streams_dashboard_vulnerability_high_link_clicked |