Add latest changes from gitlab-org/gitlab@master

1 files changed, 113 insertions, 281 deletions

diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb
index 21117f11f63..fb60ac955de 100644
--- a/spec/lib/gitlab/usage_data_spec.rb
+++ b/spec/lib/gitlab/usage_data_spec.rb
@@ -2,299 +2,127 @@
 
 require 'spec_helper'
 
-describe Gitlab::UsageData do
-  let(:projects) { create_list(:project, 4) }
-  let!(:board) { create(:board, project: projects[0]) }
+describe Gitlab::UsageData, :aggregate_failures do
+  include UsageDataHelpers
 
   before do
     allow(ActiveRecord::Base.connection).to receive(:transaction_open?).and_return(false)
   end
-  [true, false].each do |usage_ping_batch_counter_on|
-    describe "when the feature flag usage_ping_batch_counter is set to #{usage_ping_batch_counter_on}" do
+
+  shared_examples "usage data execution" do
+    describe '#data' do
+      let!(:ud) { build(:usage_data) }
+
       before do
-        stub_feature_flags(usage_ping_batch_counter: usage_ping_batch_counter_on)
+        allow(Gitlab::GrafanaEmbedUsageData).to receive(:issue_count).and_return(2)
       end
 
-      describe '#data' do
-        before do
-          create(:jira_service, project: projects[0])
-          create(:jira_service, :without_properties_callback, project: projects[1])
-          create(:jira_service, :jira_cloud_service, project: projects[2])
-          create(:jira_service, :without_properties_callback, project: projects[3],
-                 properties: { url: 'https://mysite.atlassian.net' })
-          create(:prometheus_service, project: projects[1])
-          create(:service, project: projects[0], type: 'SlackSlashCommandsService', active: true)
-          create(:service, project: projects[1], type: 'SlackService', active: true)
-          create(:service, project: projects[2], type: 'SlackService', active: true)
-          create(:service, project: projects[2], type: 'MattermostService', active: false)
-          create(:service, :template, type: 'MattermostService', active: true)
-          create(:service, project: projects[2], type: 'CustomIssueTrackerService', active: true)
-          create(:project_error_tracking_setting, project: projects[0])
-          create(:project_error_tracking_setting, project: projects[1], enabled: false)
-          create(:alerts_service, project: projects[0])
-          create(:alerts_service, :inactive, project: projects[1])
-          create_list(:issue, 2, project: projects[0], author: User.alert_bot)
-          create_list(:issue, 2, project: projects[1], author: User.alert_bot)
-          create_list(:issue, 4, project: projects[0])
-          create(:zoom_meeting, project: projects[0], issue: projects[0].issues[0], issue_status: :added)
-          create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[1], issue_status: :removed)
-          create(:zoom_meeting, project: projects[0], issue: projects[0].issues[2], issue_status: :added)
-          create_list(:zoom_meeting, 2, project: projects[0], issue: projects[0].issues[2], issue_status: :removed)
-          create(:sentry_issue, issue: projects[0].issues[0])
-
-          # Enabled clusters
-          gcp_cluster = create(:cluster_provider_gcp, :created).cluster
-          create(:cluster_provider_aws, :created)
-          create(:cluster_platform_kubernetes)
-          create(:cluster, :group)
-
-          # Disabled clusters
-          create(:cluster, :disabled)
-          create(:cluster, :group, :disabled)
-          create(:cluster, :group, :disabled)
-
-          # Applications
-          create(:clusters_applications_helm, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_ingress, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_cert_manager, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_prometheus, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_crossplane, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_runner, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_knative, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_elastic_stack, :installed, cluster: gcp_cluster)
-          create(:clusters_applications_jupyter, :installed, cluster: gcp_cluster)
-
-          create(:grafana_integration, project: projects[0], enabled: true)
-          create(:grafana_integration, project: projects[1], enabled: true)
-          create(:grafana_integration, project: projects[2], enabled: false)
-
-          allow(Gitlab::GrafanaEmbedUsageData).to receive(:issue_count).and_return(2)
-
-          ProjectFeature.first.update_attribute('repository_access_level', 0)
-        end
+      subject { described_class.data }
 
-        subject { described_class.data }
-
-        it 'gathers usage data', :aggregate_failures do
-          expect(subject.keys).to include(*%i(
-            active_user_count
-            counts
-            recorded_at
-            edition
-            version
-            installation_type
-            uuid
-            hostname
-            mattermost_enabled
-            signup_enabled
-            ldap_enabled
-            gravatar_enabled
-            omniauth_enabled
-            reply_by_email_enabled
-            container_registry_enabled
-            dependency_proxy_enabled
-            gitlab_shared_runners_enabled
-            gitlab_pages
-            git
-            gitaly
-            database
-            avg_cycle_analytics
-            influxdb_metrics_enabled
-            prometheus_metrics_enabled
-            web_ide_clientside_preview_enabled
-            ingress_modsecurity_enabled
-          ))
-        end
+      it 'gathers usage data' do
+        expect(subject.keys).to include(*UsageDataHelpers::USAGE_DATA_KEYS)
+      end
 
-        it 'gathers usage counts' do
-          smau_keys = %i(
-            snippet_create
-            snippet_update
-            snippet_comment
-            merge_request_comment
-            merge_request_create
-            commit_comment
-            wiki_pages_create
-            wiki_pages_update
-            wiki_pages_delete
-            web_ide_views
-            web_ide_commits
-            web_ide_merge_requests
-            web_ide_previews
-            navbar_searches
-            cycle_analytics_views
-            productivity_analytics_views
-            source_code_pushes
-          )
+      it 'gathers usage counts' do
+        count_data = subject[:counts]
 
-          expected_keys = %i(
-            assignee_lists
-            boards
-            ci_builds
-            ci_internal_pipelines
-            ci_external_pipelines
-            ci_pipeline_config_auto_devops
-            ci_pipeline_config_repository
-            ci_runners
-            ci_triggers
-            ci_pipeline_schedules
-            auto_devops_enabled
-            auto_devops_disabled
-            deploy_keys
-            deployments
-            successful_deployments
-            failed_deployments
-            environments
-            clusters
-            clusters_enabled
-            project_clusters_enabled
-            group_clusters_enabled
-            clusters_disabled
-            project_clusters_disabled
-            group_clusters_disabled
-            clusters_platforms_eks
-            clusters_platforms_gke
-            clusters_platforms_user
-            clusters_applications_helm
-            clusters_applications_ingress
-            clusters_applications_cert_managers
-            clusters_applications_prometheus
-            clusters_applications_crossplane
-            clusters_applications_runner
-            clusters_applications_knative
-            clusters_applications_elastic_stack
-            clusters_applications_jupyter
-            in_review_folder
-            grafana_integrated_projects
-            groups
-            issues
-            issues_created_from_gitlab_error_tracking_ui
-            issues_with_associated_zoom_link
-            issues_using_zoom_quick_actions
-            issues_with_embedded_grafana_charts_approx
-            incident_issues
-            keys
-            label_lists
-            labels
-            lfs_objects
-            merge_requests
-            milestone_lists
-            milestones
-            notes
-            pool_repositories
-            projects
-            projects_imported_from_github
-            projects_asana_active
-            projects_jira_active
-            projects_jira_server_active
-            projects_jira_cloud_active
-            projects_slack_notifications_active
-            projects_slack_slash_active
-            projects_slack_active
-            projects_slack_slash_commands_active
-            projects_custom_issue_tracker_active
-            projects_mattermost_active
-            projects_prometheus_active
-            projects_with_repositories_enabled
-            projects_with_error_tracking_enabled
-            projects_with_alerts_service_enabled
-            pages_domains
-            protected_branches
-            releases
-            remote_mirrors
-            snippets
-            suggestions
-            todos
-            uploads
-            web_hooks
-          ).push(*smau_keys)
-
-          count_data = subject[:counts]
-
-          expect(count_data[:boards]).to eq(1)
-          expect(count_data[:projects]).to eq(4)
-          expect(count_data.values_at(*smau_keys)).to all(be_an(Integer))
-          expect(count_data.keys).to include(*expected_keys)
-          expect(expected_keys - count_data.keys).to be_empty
-        end
+        expect(count_data[:boards]).to eq(1)
+        expect(count_data[:projects]).to eq(4)
+        expect(count_data.values_at(*UsageDataHelpers::SMAU_KEYS)).to all(be_an(Integer))
+        expect(count_data.keys).to include(*UsageDataHelpers::COUNTS_KEYS)
+        expect(UsageDataHelpers::COUNTS_KEYS - count_data.keys).to be_empty
+      end
 
-        it 'gathers projects data correctly', :aggregate_failures do
-          count_data = subject[:counts]
-
-          expect(count_data[:projects]).to eq(4)
-          expect(count_data[:projects_asana_active]).to eq(0)
-          expect(count_data[:projects_prometheus_active]).to eq(1)
-          expect(count_data[:projects_jira_active]).to eq(4)
-          expect(count_data[:projects_jira_server_active]).to eq(2)
-          expect(count_data[:projects_jira_cloud_active]).to eq(2)
-          expect(count_data[:projects_slack_notifications_active]).to eq(2)
-          expect(count_data[:projects_slack_slash_active]).to eq(1)
-          expect(count_data[:projects_slack_active]).to eq(2)
-          expect(count_data[:projects_slack_slash_commands_active]).to eq(1)
-          expect(count_data[:projects_custom_issue_tracker_active]).to eq(1)
-          expect(count_data[:projects_mattermost_active]).to eq(0)
-          expect(count_data[:projects_with_repositories_enabled]).to eq(3)
-          expect(count_data[:projects_with_error_tracking_enabled]).to eq(1)
-          expect(count_data[:projects_with_alerts_service_enabled]).to eq(1)
-          expect(count_data[:issues_created_from_gitlab_error_tracking_ui]).to eq(1)
-          expect(count_data[:issues_with_associated_zoom_link]).to eq(2)
-          expect(count_data[:issues_using_zoom_quick_actions]).to eq(3)
-          expect(count_data[:issues_with_embedded_grafana_charts_approx]).to eq(2)
-          expect(count_data[:incident_issues]).to eq(4)
-
-          expect(count_data[:clusters_enabled]).to eq(4)
-          expect(count_data[:project_clusters_enabled]).to eq(3)
-          expect(count_data[:group_clusters_enabled]).to eq(1)
-          expect(count_data[:clusters_disabled]).to eq(3)
-          expect(count_data[:project_clusters_disabled]).to eq(1)
-          expect(count_data[:group_clusters_disabled]).to eq(2)
-          expect(count_data[:group_clusters_enabled]).to eq(1)
-          expect(count_data[:clusters_platforms_eks]).to eq(1)
-          expect(count_data[:clusters_platforms_gke]).to eq(1)
-          expect(count_data[:clusters_platforms_user]).to eq(1)
-          expect(count_data[:clusters_applications_helm]).to eq(1)
-          expect(count_data[:clusters_applications_ingress]).to eq(1)
-          expect(count_data[:clusters_applications_cert_managers]).to eq(1)
-          expect(count_data[:clusters_applications_crossplane]).to eq(1)
-          expect(count_data[:clusters_applications_prometheus]).to eq(1)
-          expect(count_data[:clusters_applications_runner]).to eq(1)
-          expect(count_data[:clusters_applications_knative]).to eq(1)
-          expect(count_data[:clusters_applications_elastic_stack]).to eq(1)
-          expect(count_data[:grafana_integrated_projects]).to eq(2)
-          expect(count_data[:clusters_applications_jupyter]).to eq(1)
-        end
+      it 'gathers projects data correctly' do
+        count_data = subject[:counts]
+
+        expect(count_data[:projects]).to eq(4)
+        expect(count_data[:projects_asana_active]).to eq(0)
+        expect(count_data[:projects_prometheus_active]).to eq(1)
+        expect(count_data[:projects_jira_active]).to eq(4)
+        expect(count_data[:projects_jira_server_active]).to eq(2)
+        expect(count_data[:projects_jira_cloud_active]).to eq(2)
+        expect(count_data[:projects_slack_notifications_active]).to eq(2)
+        expect(count_data[:projects_slack_slash_active]).to eq(1)
+        expect(count_data[:projects_slack_active]).to eq(2)
+        expect(count_data[:projects_slack_slash_commands_active]).to eq(1)
+        expect(count_data[:projects_custom_issue_tracker_active]).to eq(1)
+        expect(count_data[:projects_mattermost_active]).to eq(0)
+        expect(count_data[:projects_with_repositories_enabled]).to eq(3)
+        expect(count_data[:projects_with_error_tracking_enabled]).to eq(1)
+        expect(count_data[:projects_with_alerts_service_enabled]).to eq(1)
+        expect(count_data[:issues_created_from_gitlab_error_tracking_ui]).to eq(1)
+        expect(count_data[:issues_with_associated_zoom_link]).to eq(2)
+        expect(count_data[:issues_using_zoom_quick_actions]).to eq(3)
+        expect(count_data[:issues_with_embedded_grafana_charts_approx]).to eq(2)
+        expect(count_data[:incident_issues]).to eq(4)
+
+        expect(count_data[:clusters_enabled]).to eq(4)
+        expect(count_data[:project_clusters_enabled]).to eq(3)
+        expect(count_data[:group_clusters_enabled]).to eq(1)
+        expect(count_data[:clusters_disabled]).to eq(3)
+        expect(count_data[:project_clusters_disabled]).to eq(1)
+        expect(count_data[:group_clusters_disabled]).to eq(2)
+        expect(count_data[:group_clusters_enabled]).to eq(1)
+        expect(count_data[:clusters_platforms_eks]).to eq(1)
+        expect(count_data[:clusters_platforms_gke]).to eq(1)
+        expect(count_data[:clusters_platforms_user]).to eq(1)
+        expect(count_data[:clusters_applications_helm]).to eq(1)
+        expect(count_data[:clusters_applications_ingress]).to eq(1)
+        expect(count_data[:clusters_applications_cert_managers]).to eq(1)
+        expect(count_data[:clusters_applications_crossplane]).to eq(1)
+        expect(count_data[:clusters_applications_prometheus]).to eq(1)
+        expect(count_data[:clusters_applications_runner]).to eq(1)
+        expect(count_data[:clusters_applications_knative]).to eq(1)
+        expect(count_data[:clusters_applications_elastic_stack]).to eq(1)
+        expect(count_data[:grafana_integrated_projects]).to eq(2)
+        expect(count_data[:clusters_applications_jupyter]).to eq(1)
+      end
 
-        it 'works when queries time out' do
-          allow_any_instance_of(ActiveRecord::Relation)
-            .to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
+      it 'works when queries time out' do
+        allow_any_instance_of(ActiveRecord::Relation)
+          .to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
 
-          expect { subject }.not_to raise_error
-        end
+        expect { subject }.not_to raise_error
       end
+    end
 
-      describe '#usage_data_counters' do
-        subject { described_class.usage_data_counters }
+    describe '#usage_data_counters' do
+      subject { described_class.usage_data_counters }
 
-        it { is_expected.to all(respond_to :totals) }
+      it { is_expected.to all(respond_to :totals) }
 
-        describe 'the results of calling #totals on all objects in the array' do
-          subject { described_class.usage_data_counters.map(&:totals) }
+      describe 'the results of calling #totals on all objects in the array' do
+        subject { described_class.usage_data_counters.map(&:totals) }
 
-          it { is_expected.to all(be_a Hash) }
-          it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) }
-        end
+        it { is_expected.to all(be_a Hash) }
+        it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) }
+      end
 
-        it 'does not have any conflicts' do
-          all_keys = subject.flat_map { |counter| counter.totals.keys }
+      it 'does not have any conflicts' do
+        all_keys = subject.flat_map { |counter| counter.totals.keys }
 
-          expect(all_keys.size).to eq all_keys.to_set.size
-        end
+        expect(all_keys.size).to eq all_keys.to_set.size
       end
+    end
 
+    describe '#license_usage_data' do
+      subject { described_class.license_usage_data }
+
+      it 'gathers license data' do
+        expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
+        expect(subject[:version]).to eq(Gitlab::VERSION)
+        expect(subject[:installation_type]).to eq('gitlab-development-kit')
+        expect(subject[:active_user_count]).to eq(User.active.size)
+        expect(subject[:recorded_at]).to be_a(Time)
+      end
+    end
+
+    context 'when not relying on database records' do
       describe '#features_usage_data_ce' do
         subject { described_class.features_usage_data_ce }
 
-        it 'gathers feature usage data', :aggregate_failures do
+        it 'gathers feature usage data' do
           expect(subject[:mattermost_enabled]).to eq(Gitlab.config.mattermost.enabled)
           expect(subject[:signup_enabled]).to eq(Gitlab::CurrentSettings.allow_signup?)
           expect(subject[:ldap_enabled]).to eq(Gitlab.config.ldap.enabled)
@@ -311,7 +139,7 @@ describe Gitlab::UsageData do
       describe '#components_usage_data' do
         subject { described_class.components_usage_data }
 
-        it 'gathers components usage data', :aggregate_failures do
+        it 'gathers components usage data' do
           expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled)
           expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION)
           expect(subject[:git][:version]).to eq(Gitlab::Git.version)
@@ -360,18 +188,6 @@ describe Gitlab::UsageData do
         end
       end
 
-      describe '#license_usage_data' do
-        subject { described_class.license_usage_data }
-
-        it 'gathers license data', :aggregate_failures do
-          expect(subject[:uuid]).to eq(Gitlab::CurrentSettings.uuid)
-          expect(subject[:version]).to eq(Gitlab::VERSION)
-          expect(subject[:installation_type]).to eq('gitlab-development-kit')
-          expect(subject[:active_user_count]).to eq(User.active.count)
-          expect(subject[:recorded_at]).to be_a(Time)
-        end
-      end
-
       describe '#count' do
         let(:relation) { double(:relation) }
 
@@ -405,4 +221,20 @@ describe Gitlab::UsageData do
       end
     end
   end
+
+  context 'when usage usage_ping_batch_counter is true' do
+    before do
+      stub_feature_flags(usage_ping_batch_counter: true)
+    end
+
+    it_behaves_like 'usage data execution'
+  end
+
+  context 'when usage usage_ping_batch_counter is false' do
+    before do
+      stub_feature_flags(usage_ping_batch_counter: false)
+    end
+
+    it_behaves_like 'usage data execution'
+  end
 end