Add latest changes from gitlab-org/gitlab@master

2 files changed, 554 insertions, 29 deletions

diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
index c9106d7c6b8..d92eb22545d 100644
--- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
@@ -3,6 +3,7 @@
 module Gitlab::UsageDataCounters
   class CiTemplateUniqueCounter
     REDIS_SLOT = 'ci_templates'
+    KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__)
 
     # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_`
     TEMPLATE_TO_EVENT = {
@@ -20,19 +21,26 @@ module Gitlab::UsageDataCounters
 
     class << self
       def track_unique_project_event(project_id:, template:, config_source:)
-        if event = unique_project_event(template, config_source)
-          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
-        end
+        return unless TEMPLATE_TO_EVENT[template] || Feature.enabled?(:track_all_ci_template_inclusions, default_enabled: :yaml)
+
+        Gitlab::UsageDataCounters::HLLRedisCounter.track_event(ci_template_event_name(template, config_source), values: project_id)
       end
 
-      private
+      def ci_templates(relative_base = 'lib/gitlab/ci/templates')
+        Dir.glob('**/*.gitlab-ci.yml', base: Rails.root.join(relative_base))
+      end
+
+      def ci_template_event_name(template_name, config_source)
+        prefix = 'implicit_' if config_source.to_s == 'auto_devops_source'
+        template_event_name = TEMPLATE_TO_EVENT[template_name] || template_to_event_name(template_name)
 
-      def unique_project_event(template, config_source)
-        if name = TEMPLATE_TO_EVENT[template]
-          prefix = 'implicit_' if config_source.to_s == 'auto_devops_source'
+        "p_#{REDIS_SLOT}_#{prefix}#{template_event_name}"
+      end
+
+      private
 
-          "p_#{REDIS_SLOT}_#{prefix}#{name}"
-        end
+      def template_to_event_name(template)
+        ActiveSupport::Inflector.parameterize(template.chomp('.gitlab-ci.yml'), separator: '_').underscore
       end
     end
   end
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
index 3c692f2b1af..cf790767f17 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
@@ -4,73 +4,590 @@
   redis_slot: ci_templates
   aggregation: weekly
 
-- name: p_ci_templates_implicit_auto_devops_build
+# Explicit include:template pipeline events
+- name: p_ci_templates_5_min_production_app
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
 
-- name: p_ci_templates_implicit_auto_devops_deploy
+- name: p_ci_templates_aws_cf_deploy_ec2
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
 
-- name: p_ci_templates_implicit_security_sast
+- name: p_ci_templates_auto_devops_build
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
 
-- name: p_ci_templates_implicit_security_secret_detection
+- name: p_ci_templates_auto_devops_deploy
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
 
-# Explicit include:template pipeline events
-- name: p_ci_templates_5_min_production_app
+- name: p_ci_templates_auto_devops_deploy_latest
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
 
+# This part of the file is generated automatically by
+#   bin/rake gitlab:usage_data:generate_ci_template_events
+#
+# Do not edit it manually!
+#
+# The section above this should be removed once we roll out tracking all ci
+# templates
+# https://gitlab.com/gitlab-org/gitlab/-/issues/339684
+
+- name: p_ci_templates_terraform_base_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_terraform_base
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_dotnet
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_nodejs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_openshift
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_auto_devops
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_aws_cf_deploy_ec2
+- name: p_ci_templates_bash
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_aws_deploy_ecs
+- name: p_ci_templates_rust
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_auto_devops_build
+- name: p_ci_templates_elixir
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_auto_devops_deploy
+- name: p_ci_templates_clojure
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_auto_devops_deploy_latest
+- name: p_ci_templates_crystal
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_getting_started
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_code_quality
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_verify_load_performance_testing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_verify_accessibility
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_verify_failfast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_verify_browser_performance
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_verify_browser_performance_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_grails
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
 - name: p_ci_templates_security_sast
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
+- name: p_ci_templates_security_dast_runner_validation
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_dast_on_demand_scan
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_security_secret_detection
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
-
-- name: p_ci_templates_terraform_base_latest
+- name: p_ci_templates_security_license_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_coverage_fuzzing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_api_fuzzing_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_secure_binaries
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_dast_api
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_container_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_dast_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_dependency_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_api_fuzzing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_dast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_security_cluster_image_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_ios_fastlane
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_composer
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_c
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_python
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_android_fastlane
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_android_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_django
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_maven
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_flutter
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_workflows_branch_pipelines
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_workflows_mergerequest_pipelines
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_laravel
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_managed_cluster_applications
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_php
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_packer
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_terraform
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_mono
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_serverless
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_go
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_scala
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_latex
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_android
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_indeni_cloudrail
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_deploy_ecs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_aws_cf_provision_and_deploy_ec2
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_aws_deploy_ecs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_gradle
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_chef
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_dast_default_branch_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_load_performance_testing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_helm_2to3
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_code_intelligence
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_code_quality
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_deploy_ecs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_deploy_ec2
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_build
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_browser_performance_testing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_test
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_deploy_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_browser_performance_testing_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_cf_provision
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_jobs_build_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_terraform_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_swift
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_jekyll
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_harp
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_octopress
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_brunch
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_doxygen
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_hyde
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_lektor
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_jbake
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_hexo
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_middleman
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_hugo
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_pelican
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_nanoc
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_swaggerui
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_jigsaw
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_metalsmith
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_gatsby
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_pages_html
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_dart
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_docker
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_julia
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_npm
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_dotnet_core
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_5_minute_production_app
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_ruby
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_load_performance_testing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_helm_2to3
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_code_intelligence
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_code_quality
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_deploy_ecs
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_deploy_ec2
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_auto_devops_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_auto_devops_build
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_browser_performance_testing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_test
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_auto_devops_deploy_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_browser_performance_testing_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_cf_provision
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_jobs_build_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dast_runner_validation
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dast_on_demand_scan
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_license_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_coverage_fuzzing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_api_fuzzing_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_secure_binaries
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dast_api
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_container_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dast_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dependency_scanning
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_api_fuzzing
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_dast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+- name: p_ci_templates_implicit_security_cluster_image_scanning
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly