Add latest changes from gitlab-org/gitlab@master

11 files changed, 63 insertions, 29 deletions

diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb
index ab6a4e4a04a..d595b5b2e09 100644
--- a/lib/api/admin/plan_limits.rb
+++ b/lib/api/admin/plan_limits.rb
@@ -37,6 +37,7 @@ module API
 
         optional :conan_max_file_size, type: Integer, desc: 'Maximum Conan package file size in bytes'
         optional :generic_packages_max_file_size, type: Integer, desc: 'Maximum generic package file size in bytes'
+        optional :helm_max_file_size, type: Integer, desc: 'Maximum Helm chart file size in bytes'
         optional :maven_max_file_size, type: Integer, desc: 'Maximum Maven package file size in bytes'
         optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes'
         optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes'
diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb
index dabb6c7ab3a..29d02e7d1f1 100644
--- a/lib/api/ci/helpers/runner.rb
+++ b/lib/api/ci/helpers/runner.rb
@@ -52,7 +52,7 @@ module API
 
         # HTTP status codes to terminate the job on GitLab Runner:
         # - 403
-        def authenticate_job!(require_running: true)
+        def authenticate_job!(require_running: true, heartbeat_runner: false)
           job = current_job
 
           # 404 is not returned here because we want to terminate the job if it's
@@ -70,7 +70,17 @@ module API
             job_forbidden!(job, 'Job is not running') unless job.running?
           end
 
-          job.runner&.heartbeat(get_runner_ip)
+          # Only some requests (like updating the job or patching the trace) should trigger
+          # runner heartbeat. Operations like artifacts uploading are executed in context of
+          # the running job and in the job environment, which in many cases will cause the IP
+          # to be updated to not the expected value. And operations like artifacts downloads can
+          # be done even after the job is finished and from totally different runners - while
+          # they would then update the connection status of not the runner that they should.
+          # Runner requests done in context of job authentication should explicitly define when
+          # the heartbeat should be triggered.
+          if heartbeat_runner
+            job.runner&.heartbeat(get_runner_ip)
+          end
 
           job
         end
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index aabcf34952c..4317789f7aa 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -176,7 +176,7 @@ module API
           optional :exit_code, type: Integer, desc: %q(Job's exit code)
         end
         put '/:id', feature_category: :continuous_integration do
-          job = authenticate_job!
+          job = authenticate_job!(heartbeat_runner: true)
 
           Gitlab::Metrics.add_event(:update_build)
 
@@ -203,7 +203,7 @@ module API
           optional :token, type: String, desc: %q(Job's authentication token)
         end
         patch '/:id/trace', feature_category: :continuous_integration do
-          job = authenticate_job!
+          job = authenticate_job!(heartbeat_runner: true)
 
           error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
           content_range = request.headers['Content-Range']
diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb
index 04ec44b5167..9f4d1635998 100644
--- a/lib/api/entities/plan_limit.rb
+++ b/lib/api/entities/plan_limit.rb
@@ -5,6 +5,7 @@ module API
     class PlanLimit < Grape::Entity
       expose :conan_max_file_size
       expose :generic_packages_max_file_size
+      expose :helm_max_file_size
       expose :maven_max_file_size
       expose :npm_max_file_size
       expose :nuget_max_file_size
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 89fd59d98f4..f47c3decd3c 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -53,6 +53,10 @@ variables:
   # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
   # KUBE_INGRESS_BASE_DOMAIN: domain.example.com
 
+  # Allows Container-Scanning to correctly correlate image names when using Jobs/Build.gitlab-ci.yml
+  CI_APPLICATION_TAG: $CI_COMMIT_SHA
+  CS_DEFAULT_BRANCH_IMAGE: $CI_REGISTRY_IMAGE/$CI_DEFAULT_BRANCH:$CI_APPLICATION_TAG
+
   POSTGRES_USER: user
   POSTGRES_PASSWORD: testing-password
   POSTGRES_ENABLED: "true"
diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml
index dbea7c9f690..430ac785449 100644
--- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml
+++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml
@@ -1,19 +1,20 @@
-chat_names:
-  - to_table: ci_pipeline_chat_data
+ci_pipeline_chat_data:
+  - table: chat_names
     column: chat_name_id
     on_delete: async_delete
-ci_builds:
-  - to_table: dast_site_profiles_builds
+dast_scanner_profiles_builds:
+  - table: ci_builds
     column: ci_build_id
     on_delete: async_delete
-  - to_table: dast_scanner_profiles_builds
+dast_scanner_profiles_builds:
+  - table: ci_builds
     column: ci_build_id
     on_delete: async_delete
-ci_pipelines:
-  - to_table: dast_profiles_pipelines
+dast_profiles_pipelines:
+  - table: ci_pipelines
     column: ci_pipeline_id
     on_delete: async_delete
-ci_runners:
-  - to_table: clusters_applications_runners
+clusters_applications_runners:
+  - table: ci_runners
     column: runner_id
     on_delete: async_nullify
diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb
index 1d885b3fbc8..1ecfb5ce47f 100644
--- a/lib/gitlab/database/loose_foreign_keys.rb
+++ b/lib/gitlab/database/loose_foreign_keys.rb
@@ -4,25 +4,25 @@ module Gitlab
   module Database
     module LooseForeignKeys
       def self.definitions_by_table
-        @definitions_by_table ||= definitions.group_by(&:from_table).with_indifferent_access.freeze
+        @definitions_by_table ||= definitions.group_by(&:to_table).with_indifferent_access.freeze
       end
 
       def self.definitions
-        @definitions ||= loose_foreign_keys_yaml.flat_map do |parent_table_name, configs|
-          configs.map { |config| build_definition(parent_table_name, config) }
+        @definitions ||= loose_foreign_keys_yaml.flat_map do |child_table_name, configs|
+          configs.map { |config| build_definition(child_table_name, config) }
         end.freeze
       end
 
-      def self.build_definition(parent_table_name, config)
-        to_table = config.fetch('to_table')
+      def self.build_definition(child_table_name, config)
+        parent_table_name = config.fetch('table')
 
         ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(
+          child_table_name,
           parent_table_name,
-          to_table,
           {
             column: config.fetch('column'),
             on_delete: config.fetch('on_delete').to_sym,
-            gitlab_schema: GitlabSchema.table_schema(to_table)
+            gitlab_schema: GitlabSchema.table_schema(child_table_name)
           }
         )
       end
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 216b1d04bf6..a58b4beb0df 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -2,11 +2,9 @@
 
 module Gitlab
   module Tracking
-    SNOWPLOW_NAMESPACE = 'gl'
-
     class << self
       def enabled?
-        snowplow_micro_enabled? || Gitlab::CurrentSettings.snowplow_enabled?
+        snowplow.enabled?
       end
 
       def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
index 5596e9acd30..ddcd4693738 100644
--- a/lib/gitlab/tracking/destinations/snowplow.rb
+++ b/lib/gitlab/tracking/destinations/snowplow.rb
@@ -8,6 +8,8 @@ module Gitlab
       class Snowplow < Base
         extend ::Gitlab::Utils::Override
 
+        SNOWPLOW_NAMESPACE = 'gl'
+
         override :event
         def event(category, action, label: nil, property: nil, value: nil, context: nil)
           return unless enabled?
@@ -19,7 +21,7 @@ module Gitlab
         def options(group)
           additional_features = Feature.enabled?(:additional_snowplow_tracking, group, type: :ops)
           {
-            namespace: Gitlab::Tracking::SNOWPLOW_NAMESPACE,
+            namespace: SNOWPLOW_NAMESPACE,
             hostname: hostname,
             cookie_domain: cookie_domain,
             app_id: app_id,
@@ -28,16 +30,16 @@ module Gitlab
           }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
         end
 
+        def enabled?
+          Gitlab::CurrentSettings.snowplow_enabled?
+        end
+
         def hostname
           Gitlab::CurrentSettings.snowplow_collector_hostname
         end
 
         private
 
-        def enabled?
-          Gitlab::Tracking.enabled?
-        end
-
         def app_id
           Gitlab::CurrentSettings.snowplow_app_id
         end
@@ -54,7 +56,7 @@ module Gitlab
           @tracker ||= SnowplowTracker::Tracker.new(
             emitter,
             SnowplowTracker::Subject.new,
-            Gitlab::Tracking::SNOWPLOW_NAMESPACE,
+            SNOWPLOW_NAMESPACE,
             app_id
           )
         end
diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb
index 8beab910818..3553efba1e1 100644
--- a/lib/gitlab/tracking/destinations/snowplow_micro.rb
+++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb
@@ -18,6 +18,11 @@ module Gitlab
           ).transform_keys! { |key| key.to_s.camelize(:lower).to_sym }
         end
 
+        override :enabled?
+        def enabled?
+          true
+        end
+
         override :hostname
         def hostname
           "#{uri.host}:#{uri.port}"
diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml
index 261bdeb9bfa..5a1e7f03278 100644
--- a/lib/gitlab/usage_data_counters/known_events/analytics.yml
+++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml
@@ -66,3 +66,15 @@
   category: analytics
   redis_slot: analytics
   aggregation: weekly
+- name: p_analytics_ci_cd_pipelines
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+- name: p_analytics_ci_cd_deployment_frequency
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly
+- name: p_analytics_ci_cd_lead_time
+  category: analytics
+  redis_slot: analytics
+  aggregation: weekly