Add latest changes from gitlab-org/gitlab@master

10 files changed, 281 insertions, 16 deletions

diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
new file mode 100644
index 00000000000..ee459f2790a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # rubocop: disable CodeReuse/ActiveRecord
+        class BaseQueryBuilder
+          include StageQueryHelpers
+
+          MODEL_CLASSES = {
+            MergeRequest.to_s => ::Analytics::CycleAnalytics::MergeRequestStageEvent,
+            Issue.to_s => ::Analytics::CycleAnalytics::IssueStageEvent
+          }.freeze
+
+          # Allowed params:
+          # * from - stage end date filter start date
+          # * to - stage end date filter to date
+          # * author_username
+          # * milestone_title
+          # * label_name (array)
+          # * assignee_username (array)
+          # * project_ids (array)
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+            @root_ancestor = stage.parent.root_ancestor
+            @stage_event_model = MODEL_CLASSES.fetch(stage.subject_class.to_s)
+          end
+
+          def build
+            query = base_query
+            query = filter_by_stage_parent(query)
+            query = filter_author(query)
+            query = filter_milestone_ids(query)
+            query = filter_label_names(query)
+            filter_assignees(query)
+          end
+
+          def filter_author(query)
+            return query if params[:author_username].blank?
+
+            user = User.by_username(params[:author_username]).first
+
+            return query.none if user.blank?
+
+            query.authored(user)
+          end
+
+          def filter_milestone_ids(query)
+            return query if params[:milestone_title].blank?
+
+            milestone = MilestonesFinder
+              .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title])
+              .execute
+              .first
+
+            return query.none if milestone.blank?
+
+            query.with_milestone_id(milestone.id)
+          end
+
+          def filter_label_names(query)
+            return query if params[:label_name].blank?
+
+            all_label_ids = Issuables::LabelFilter
+              .new(group: root_ancestor, project: nil, params: { label_name: params[:label_name] })
+              .find_label_ids(params[:label_name])
+
+            return query.none if params[:label_name].size != all_label_ids.size
+
+            all_label_ids.each do |label_ids|
+              relation = LabelLink
+                .where(target_type: stage.subject_class.name)
+                .where(LabelLink.arel_table['target_id'].eq(query.model.arel_table[query.model.issuable_id_column]))
+
+              relation = relation.where(label_id: label_ids)
+
+              query = query.where(relation.arel.exists)
+            end
+
+            query
+          end
+
+          def filter_assignees(query)
+            return query if params[:assignee_username].blank?
+
+            Issuables::AssigneeFilter
+              .new(params: { assignee_username: params[:assignee_username] })
+              .filter(query)
+          end
+
+          def filter_by_stage_parent(query)
+            query.by_project_id(stage.parent_id)
+          end
+
+          def base_query
+            query = stage_event_model
+              .by_stage_event_hash_id(stage.stage_event_hash_id)
+
+            from = params[:from] || 30.days.ago
+            if in_progress?
+              query = query
+                .end_event_is_not_happened_yet
+                .opened_state
+                .start_event_timestamp_after(from)
+              query = query.start_event_timestamp_before(params[:to]) if params[:to]
+            else
+              query = query.end_event_timestamp_after(from)
+              query = query.end_event_timestamp_before(params[:to]) if params[:to]
+            end
+
+            query
+          end
+
+          private
+
+          attr_reader :stage, :params, :root_ancestor, :stage_event_model
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
+Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder')
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
new file mode 100644
index 00000000000..ab3ae93f5ff
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # Arguments:
+        #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+        #   params:
+        #     current_user: an instance of User
+        #     from: DateTime
+        #     to: DateTime
+        class DataCollector
+          include Gitlab::Utils::StrongMemoize
+
+          MAX_COUNT = 10001
+
+          delegate :serialized_records, to: :records_fetcher
+
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+          end
+
+          def median
+            strong_memoize(:median) { Median.new(stage: stage, query: query, params: params) }
+          end
+
+          def count
+            strong_memoize(:count) { limit_count }
+          end
+
+          private
+
+          attr_reader :stage, :params
+
+          def query
+            BaseQueryBuilder.new(stage: stage, params: params).build
+          end
+
+          def limit_count
+            query.limit(MAX_COUNT).count
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
new file mode 100644
index 00000000000..181ee20948b
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        class Median
+          include StageQueryHelpers
+
+          def initialize(stage:, query:, params:)
+            @stage = stage
+            @query = query
+            @params = params
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def seconds
+            @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil)
+            result = @query.take || {}
+
+            result['median'] || nil
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def days
+            seconds ? seconds.fdiv(1.day) : nil
+          end
+
+          private
+
+          attr_reader :stage, :query, :params
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
new file mode 100644
index 00000000000..f23d1832df9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        module StageQueryHelpers
+          def percentile_cont
+            percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+            Arel::Nodes::NamedFunction.new(
+              'percentile_cont(0.5) WITHIN GROUP',
+              [percentile_cont_ordering]
+            )
+          end
+
+          def duration
+            if in_progress?
+              Arel::Nodes::Subtraction.new(
+                Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i]),
+                query.model.arel_table[:start_event_timestamp]
+              )
+            else
+              Arel::Nodes::Subtraction.new(
+                query.model.arel_table[:end_event_timestamp],
+                query.model.arel_table[:start_event_timestamp]
+              )
+            end
+          end
+
+          def median_duration_in_seconds
+            Arel::Nodes::Extract.new(percentile_cont, :epoch)
+          end
+
+          def in_progress?
+            params[:end_event_filter] == :in_progress
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
index 56179533ffb..a26df55bd0a 100644
--- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -29,7 +29,11 @@ module Gitlab
 
         def median
           strong_memoize(:median) do
-            Median.new(stage: stage, query: query, params: params)
+            if use_aggregated_data_collector?
+              aggregated_data_collector.median
+            else
+              Median.new(stage: stage, query: query, params: params)
+            end
           end
         end
 
@@ -41,7 +45,11 @@ module Gitlab
 
         def count
           strong_memoize(:count) do
-            limit_count
+            if use_aggregated_data_collector?
+              aggregated_data_collector.count
+            else
+              limit_count
+            end
           end
         end
 
@@ -59,6 +67,14 @@ module Gitlab
         def limit_count
           query.limit(MAX_COUNT).count
         end
+
+        def aggregated_data_collector
+          @aggregated_data_collector ||= Aggregated::DataCollector.new(stage: stage, params: params)
+        end
+
+        def use_aggregated_data_collector?
+          params.fetch(:use_aggregated_data_collector, false)
+        end
       end
     end
   end
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 94e20762368..bc9d94ef09c 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -79,7 +79,8 @@ module Gitlab
             sort: sort&.to_sym,
             direction: direction&.to_sym,
             page: page,
-            end_event_filter: end_event_filter.to_sym
+            end_event_filter: end_event_filter.to_sym,
+            use_aggregated_data_collector: Feature.enabled?(:use_vsa_aggregated_tables, group || project, default_enabled: :yaml)
           }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES))
         end
 
diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb
index dc6678dc93b..c9ae7d222ed 100644
--- a/lib/gitlab/database/load_balancing/load_balancer.rb
+++ b/lib/gitlab/database/load_balancing/load_balancer.rb
@@ -54,7 +54,10 @@ module Gitlab
               connection = host.connection
               return yield connection
             rescue StandardError => error
-              if serialization_failure?(error)
+              if primary_only?
+                # If we only have primary configured, retrying is pointless
+                raise error
+              elsif serialization_failure?(error)
                 # This error can occur when a query conflicts. See
                 # https://www.postgresql.org/docs/current/static/hot-standby.html#HOT-STANDBY-CONFLICT
                 # for more information.
diff --git a/lib/gitlab/graphql/tracers/logger_tracer.rb b/lib/gitlab/graphql/tracers/logger_tracer.rb
index 8755a1844cd..c7ba56824db 100644
--- a/lib/gitlab/graphql/tracers/logger_tracer.rb
+++ b/lib/gitlab/graphql/tracers/logger_tracer.rb
@@ -45,14 +45,12 @@ module Gitlab
           ::Gitlab::GraphqlLogger.info(info)
         end
 
-        def query_variables_for_logging(query)
-          clean_variables(query.provided_variables)
-        end
-
         def clean_variables(variables)
-          ActiveSupport::ParameterFilter
+          filtered = ActiveSupport::ParameterFilter
             .new(::Rails.application.config.filter_parameters)
             .filter(variables)
+
+          filtered&.to_s
         end
       end
     end
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
index 9b6bae12057..0195de0b321 100644
--- a/lib/gitlab/subscription_portal.rb
+++ b/lib/gitlab/subscription_portal.rb
@@ -4,11 +4,7 @@ module Gitlab
   module SubscriptionPortal
     def self.default_subscriptions_url
       if ::Gitlab.dev_or_test_env?
-        if Feature.enabled?(:new_customersdot_staging_url, default_enabled: :yaml)
-          'https://customers.staging.gitlab.com'
-        else
-          'https://customers.stg.gitlab.com'
-        end
+        'https://customers.staging.gitlab.com'
       else
         'https://customers.gitlab.com'
       end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index c40aa2273aa..3a905a2e1c5 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -8,6 +8,7 @@ require 'uri'
 module Gitlab
   class Workhorse
     SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'
+    SEND_DEPENDENCY_CONTENT_TYPE_HEADER = 'Workhorse-Proxy-Content-Type'
     VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'
     INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'
@@ -170,9 +171,9 @@ module Gitlab
         ]
       end
 
-      def send_dependency(token, url)
+      def send_dependency(headers, url)
         params = {
-          'Header' => { Authorization: ["Bearer #{token}"] },
+          'Header' => headers,
           'Url' => url
         }