diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /lib/gitlab/usage_data.rb | |
parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to 'lib/gitlab/usage_data.rb')
-rw-r--r-- | lib/gitlab/usage_data.rb | 363 |
1 files changed, 306 insertions, 57 deletions
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 7b6f5e69ee1..9d7e6536608 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -18,7 +18,6 @@ module Gitlab class << self include Gitlab::Utils::UsageData include Gitlab::Utils::StrongMemoize - include Gitlab::UsageDataConcerns::Topology def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do @@ -27,16 +26,21 @@ module Gitlab end def uncached_data - clear_memoized_limits - - license_usage_data - .merge(system_usage_data) - .merge(features_usage_data) - .merge(components_usage_data) - .merge(cycle_analytics_usage_data) - .merge(object_store_usage_data) - .merge(topology_usage_data) - .merge(recording_ce_finish_data) + clear_memoized + + with_finished_at(:recording_ce_finished_at) do + license_usage_data + .merge(system_usage_data) + .merge(system_usage_data_monthly) + .merge(features_usage_data) + .merge(components_usage_data) + .merge(cycle_analytics_usage_data) + .merge(object_store_usage_data) + .merge(topology_usage_data) + .merge(usage_activity_by_stage) + .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period)) + .merge(analytics_unique_visits_data) + end end def to_json(force_refresh: false) @@ -59,17 +63,11 @@ module Gitlab Time.now end - def recording_ce_finish_data - { - recording_ce_finished_at: Time.now - } - end - # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - alert_bot_incident_count = count(::Issue.authored(::User.alert_bot)) - issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)) + alert_bot_incident_count = count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) + issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) { counts: { @@ -86,9 +84,9 @@ module Gitlab auto_devops_enabled: count(::ProjectAutoDevops.enabled), auto_devops_disabled: count(::ProjectAutoDevops.disabled), deploy_keys: count(DeployKey), - deployments: count(Deployment), - successful_deployments: count(Deployment.success), - failed_deployments: count(Deployment.failed), + deployments: deployment_count(Deployment), + successful_deployments: deployment_count(Deployment.success), + failed_deployments: deployment_count(Deployment.failed), environments: count(::Environment), clusters: count(::Clusters::Cluster), clusters_enabled: count(::Clusters::Cluster.enabled), @@ -111,11 +109,12 @@ module Gitlab clusters_applications_knative: count(::Clusters::Applications::Knative.available), clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available), clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available), + clusters_applications_cilium: count(::Clusters::Applications::Cilium.available), clusters_management_project: count(::Clusters::Cluster.with_management_project), in_review_folder: count(::Environment.in_review_folder), grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), - issues: count(Issue), + issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), @@ -125,7 +124,7 @@ module Gitlab issues_created_manually_from_alerts: issues_created_manually_from_alerts, incident_issues: alert_bot_incident_count, alert_bot_incident_issues: alert_bot_incident_count, - incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)), + incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -144,7 +143,6 @@ module Gitlab protected_branches: count(ProtectedBranch), releases: count(Release), remote_mirrors: count(RemoteMirror), - snippets: count(Snippet), personal_snippets: count(PersonalSnippet), project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), @@ -161,14 +159,29 @@ module Gitlab usage_counters, user_preferences_usage, ingress_modsecurity_usage, - container_expiration_policies_usage, - merge_requests_usage(default_time_period) - ) + container_expiration_policies_usage + ).tap do |data| + data[:snippets] = data[:personal_snippets] + data[:project_snippets] + end } end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: enable Metrics/AbcSize + def system_usage_data_monthly + { + counts_monthly: { + deployments: deployment_count(Deployment.where(last_28_days_time_period)), + successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)), + failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)), + personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), + project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) + }.tap do |data| + data[:snippets] = data[:personal_snippets] + data[:project_snippets] + end + } + end + # rubocop: enable CodeReuse/ActiveRecord + def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json rescue ActiveRecord::StatementInvalid @@ -197,6 +210,7 @@ module Gitlab ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled }, mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled }, omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? }, + prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, @@ -290,6 +304,10 @@ module Gitlab } end + def topology_usage_data + Gitlab::UsageData::Topology.new.topology_usage_data + end + def ingress_modsecurity_usage ## # This method measures usage of the Modsecurity Web Application Firewall across the entire @@ -336,15 +354,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage - results = Service.available_services_names.without('jira').each_with_object({}) do |service_name, response| + Service.available_services_names.without('jira').each_with_object({}) do |service_name, response| response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize)) - end - - # Keep old Slack keys for backward compatibility, https://gitlab.com/gitlab-data/analytics/issues/3241 - results[:projects_slack_notifications_active] = results[:projects_slack_active] - results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active] - - results.merge(jira_usage).merge(jira_import_usage) + end.merge(jira_usage).merge(jira_import_usage) end def jira_usage @@ -357,18 +369,15 @@ module Gitlab projects_jira_active: 0 } - Service.active - .by_type(:JiraService) - .includes(:jira_tracker_data) - .find_in_batches(batch_size: BATCH_SIZE) do |services| + JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: BATCH_SIZE) do |services| counts = services.group_by do |service| # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 service_url = service.data_fields&.url || (service.properties && service.properties['url']) service_url&.include?('.atlassian.net') ? :cloud : :server end - results[:projects_jira_server_active] += counts[:server].count if counts[:server] - results[:projects_jira_cloud_active] += counts[:cloud].count if counts[:cloud] + results[:projects_jira_server_active] += counts[:server].size if counts[:server] + results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] results[:projects_jira_active] += services.size end @@ -400,23 +409,18 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def merge_requests_usage(time_period) + def merge_requests_users(time_period) query = Event .where(target_type: Event::TARGET_TYPES[:merge_request].to_s) .where(time_period) - merge_request_users = distinct_count( + distinct_count( query, :author_id, - batch_size: 5_000, # Based on query performance, this is the optimal batch size. - start: User.minimum(:id), - finish: User.maximum(:id) + start: user_minimum_id, + finish: user_maximum_id ) - - { - merge_requests_users: merge_request_users - } end # rubocop: enable CodeReuse/ActiveRecord @@ -428,19 +432,207 @@ module Gitlab end end - def default_time_period + def last_28_days_time_period { created_at: 28.days.ago..Time.current } end + # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv + def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) + { + key => { + configure: usage_activity_by_stage_configure(time_period), + create: usage_activity_by_stage_create(time_period), + manage: usage_activity_by_stage_manage(time_period), + monitor: usage_activity_by_stage_monitor(time_period), + package: usage_activity_by_stage_package(time_period), + plan: usage_activity_by_stage_plan(time_period), + release: usage_activity_by_stage_release(time_period), + secure: usage_activity_by_stage_secure(time_period), + verify: usage_activity_by_stage_verify(time_period) + } + } + end + + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_configure(time_period) + { + clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period), + clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period), + clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period), + clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period), + clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period), + clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period), + clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period), + clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period), + clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period), + clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period), + instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period), + instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period), + group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period), + group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period), + project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period), + project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period) + } + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_create(time_period) + { + deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id), + keys: distinct_count(::Key.regular_keys.where(time_period), :user_id), + merge_requests: distinct_count(::MergeRequest.where(time_period), :author_id), + projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), + projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), + remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), + snippets: distinct_count(::Snippet.where(time_period), :author_id) + }.tap do |h| + if time_period.present? + h[:merge_requests_users] = merge_requests_users(time_period) + h.merge!(action_monthly_active_users(time_period)) + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links` + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_manage(time_period) + { + events: distinct_count(::Event.where(time_period), :author_id), + groups: distinct_count(::GroupMember.where(time_period), :user_id), + users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), + omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' } + } + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_monitor(time_period) + { + clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id), + clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), + operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), + start: user_minimum_id, + finish: user_maximum_id) + } + end + # rubocop: enable CodeReuse/ActiveRecord + + def usage_activity_by_stage_package(time_period) + {} + end + + # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads` + # Omitted because too expensive: `epics_deepest_relationship_level` + # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_plan(time_period) + { + issues: distinct_count(::Issue.where(time_period), :author_id), + notes: distinct_count(::Note.where(time_period), :author_id), + projects: distinct_count(::Project.where(time_period), :creator_id), + todos: distinct_count(::Todo.where(time_period), :author_id) + } + end + # rubocop: enable CodeReuse/ActiveRecord + + # Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains` + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_release(time_period) + { + deployments: distinct_count(::Deployment.where(time_period), :user_id), + failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id), + releases: distinct_count(::Release.where(time_period), :author_id), + successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id) + } + end + # rubocop: enable CodeReuse/ActiveRecord + + # Omitted because no user, creator or author associated: `ci_runners` + # rubocop: disable CodeReuse/ActiveRecord + def usage_activity_by_stage_verify(time_period) + { + ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id), + ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id), + ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id), + clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period) + } + end + # rubocop: enable CodeReuse/ActiveRecord + + # Currently too complicated and to get reliable counts for these stats: + # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs + # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable + def usage_activity_by_stage_secure(time_period) + {} + end + + def analytics_unique_visits_data + results = ::Gitlab::Analytics::UniqueVisits::TARGET_IDS.each_with_object({}) do |target_id, hash| + hash[target_id] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_target(target_id) } + end + results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_any_target } + + { analytics_unique_visits: results } + end + + def action_monthly_active_users(time_period) + return {} unless Feature.enabled?(Gitlab::UsageDataCounters::TrackUniqueActions::FEATURE_FLAG) + + counter = Gitlab::UsageDataCounters::TrackUniqueActions + + project_count = redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) + end + + design_count = redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueActions::DESIGN_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) + end + + wiki_count = redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueActions::WIKI_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) + end + + { + action_monthly_active_users_project_repo: project_count, + action_monthly_active_users_design_management: design_count, + action_monthly_active_users_wiki_repo: wiki_count + } + end + private + def unique_visit_service + strong_memoize(:unique_visit_service) do + ::Gitlab::Analytics::UniqueVisits.new + end + end + def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. [ - count(Issue.with_alert_management_alerts), - count(::Issue.with_self_managed_prometheus_alert_events), - count(::Issue.with_prometheus_alert_events) + count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), + count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id), + count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id) ].reduce(:+) end @@ -456,9 +648,66 @@ module Gitlab end end - def clear_memoized_limits + def issue_minimum_id + strong_memoize(:issue_minimum_id) do + ::Issue.minimum(:id) + end + end + + def issue_maximum_id + strong_memoize(:issue_maximum_id) do + ::Issue.maximum(:id) + end + end + + def deployment_minimum_id + strong_memoize(:deployment_minimum_id) do + ::Deployment.minimum(:id) + end + end + + def deployment_maximum_id + strong_memoize(:deployment_maximum_id) do + ::Deployment.maximum(:id) + end + end + + def clear_memoized + clear_memoization(:issue_minimum_id) + clear_memoization(:issue_maximum_id) clear_memoization(:user_minimum_id) clear_memoization(:user_maximum_id) + clear_memoization(:unique_visit_service) + clear_memoization(:deployment_minimum_id) + clear_memoization(:deployment_maximum_id) + clear_memoization(:approval_merge_request_rule_minimum_id) + clear_memoization(:approval_merge_request_rule_maximum_id) + end + + # rubocop: disable CodeReuse/ActiveRecord + def cluster_applications_user_distinct_count(applications, time_period) + distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id') + end + + def clusters_user_distinct_count(clusters, time_period) + distinct_count(clusters.where(time_period), :user_id) + end + # rubocop: enable CodeReuse/ActiveRecord + + def omniauth_provider_names + ::Gitlab.config.omniauth.providers.map(&:name) + end + + # LDAP provider names are set by customers and could include + # sensitive info (server names, etc). LDAP providers normally + # don't appear in omniauth providers but filter to ensure + # no internal details leak via usage ping. + def filtered_omniauth_provider_names + omniauth_provider_names.reject { |name| name.starts_with?('ldap') } + end + + def deployment_count(relation) + count relation, start: deployment_minimum_id, finish: deployment_maximum_id end end end |