Add latest changes from gitlab-org/gitlab@master

11 files changed, 215 insertions, 36 deletions

diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb
index 4ddc7e02d1b..93baf1e596c 100644
--- a/lib/gitlab/cross_project_access.rb
+++ b/lib/gitlab/cross_project_access.rb
@@ -18,7 +18,7 @@ module Gitlab
     end
 
     def add_check(
-          klass,
+      klass,
           actions: {},
           positive_condition: nil,
           negative_condition: nil,
diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb
index af363705bed..f941c57a6dd 100644
--- a/lib/gitlab/data_builder/push.rb
+++ b/lib/gitlab/data_builder/push.rb
@@ -86,7 +86,7 @@ module Gitlab
       #
       # rubocop:disable Metrics/ParameterLists
       def build(
-          project:, user:, ref:, oldrev: nil, newrev: nil,
+        project:, user:, ref:, oldrev: nil, newrev: nil,
           commits: [], commits_count: nil, message: nil, push_options: {},
           with_changed_files: true)
 
diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb
new file mode 100644
index 00000000000..9ec9ae684a5
--- /dev/null
+++ b/lib/gitlab/database/partitioning/partition_monitoring.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module Partitioning
+      class PartitionMonitoring
+        attr_reader :models
+
+        def initialize(models = PartitionCreator.models)
+          @models = models
+        end
+
+        def report_metrics
+          models.each do |model|
+            strategy = model.partitioning_strategy
+
+            gauge_present.set({ table: model.table_name }, strategy.current_partitions.size)
+            gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size)
+          end
+        end
+
+        private
+
+        def gauge_present
+          @gauge_present ||= Gitlab::Metrics.gauge(:db_partitions_present, 'Number of database partitions present')
+        end
+
+        def gauge_missing
+          @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 63cf80a10ed..40dee0142b9 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -31,8 +31,8 @@ module Gitlab
     def lexer
       @lexer ||= custom_language || begin
         Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new
-      rescue Rouge::Guesser::Ambiguous => e
-        e.alternatives.min_by(&:tag)
+                                    rescue Rouge::Guesser::Ambiguous => e
+                                      e.alternatives.min_by(&:tag)
       end
     end
 
diff --git a/lib/gitlab/search/recent_issues.rb b/lib/gitlab/search/recent_issues.rb
new file mode 100644
index 00000000000..755ca35bd66
--- /dev/null
+++ b/lib/gitlab/search/recent_issues.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Search
+    class RecentIssues
+      ITEMS_LIMIT = 100
+      EXPIRES_AFTER = 7.days
+
+      def initialize(user:, items_limit: ITEMS_LIMIT, expires_after: EXPIRES_AFTER)
+        @user = user
+        @items_limit = items_limit
+        @expires_after = expires_after
+      end
+
+      def log_view(issue)
+        return unless recent_items_enabled?
+
+        with_redis do |redis|
+          redis.zadd(key, Time.now.to_f, issue.id)
+          redis.expire(key, @expires_after)
+
+          # There is a race condition here where we could end up removing an
+          # item from 2 places concurrently but this is fine since worst case
+          # scenario we remove an extra item from the end of the list.
+          if redis.zcard(key) > @items_limit
+            redis.zremrangebyrank(key, 0, 0) # Remove least recent
+          end
+        end
+      end
+
+      def search(term)
+        return Issue.none unless recent_items_enabled?
+
+        ids = with_redis do |redis|
+          redis.zrevrange(key, 0, @items_limit - 1)
+        end.map(&:to_i)
+
+        IssuesFinder.new(@user, search: term, in: 'title').execute.reorder(nil).id_in_ordered(ids) # rubocop: disable CodeReuse/ActiveRecord
+      end
+
+      private
+
+      def with_redis(&blk)
+        Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord
+      end
+
+      def key
+        "recent_items:#{type.name.downcase}:#{@user.id}"
+      end
+
+      def type
+        Issue
+      end
+
+      def recent_items_enabled?
+        Feature.enabled?(:recent_items_search, @user)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/except.rb b/lib/gitlab/sql/except.rb
new file mode 100644
index 00000000000..82cbfa8d4ab
--- /dev/null
+++ b/lib/gitlab/sql/except.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SQL
+    # Class for building SQL EXCEPT statements.
+    #
+    # ORDER BYs are dropped from the relations as the final sort order is not
+    # guaranteed any way.
+    #
+    # Example usage:
+    #
+    #     except = Gitlab::SQL::Except.new([user.projects, user.personal_projects])
+    #     sql    = except.to_sql
+    #
+    #     Project.where("id IN (#{sql})")
+    class Except < SetOperator
+      def self.operator_keyword
+        'EXCEPT'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/intersect.rb b/lib/gitlab/sql/intersect.rb
new file mode 100644
index 00000000000..c661db3d4c5
--- /dev/null
+++ b/lib/gitlab/sql/intersect.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SQL
+    # Class for building SQL INTERSECT statements.
+    #
+    # ORDER BYs are dropped from the relations as the final sort order is not
+    # guaranteed any way.
+    #
+    # Example usage:
+    #
+    #     hierarchies = [group1.self_and_hierarchy, group2.self_and_hierarchy]
+    #     intersect   = Gitlab::SQL::Intersect.new(hierarchies)
+    #     sql         = intersect.to_sql
+    #
+    #     Project.where("id IN (#{sql})")
+    class Intersect < SetOperator
+      def self.operator_keyword
+        'INTERSECT'
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb
new file mode 100644
index 00000000000..d58a1415493
--- /dev/null
+++ b/lib/gitlab/sql/set_operator.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SQL
+    # Class for building SQL set operator statements (UNION, INTERSECT, and
+    # EXCEPT).
+    #
+    # ORDER BYs are dropped from the relations as the final sort order is not
+    # guaranteed any way.
+    #
+    # Example usage:
+    #
+    #     union = Gitlab::SQL::Union.new([user.personal_projects, user.projects])
+    #     sql   = union.to_sql
+    #
+    #     Project.where("id IN (#{sql})")
+    class SetOperator
+      def initialize(relations, remove_duplicates: true)
+        @relations = relations
+        @remove_duplicates = remove_duplicates
+      end
+
+      def self.operator_keyword
+        raise NotImplementedError
+      end
+
+      def to_sql
+        # Some relations may include placeholders for prepared statements, these
+        # aren't incremented properly when joining relations together this way.
+        # By using "unprepared_statements" we remove the usage of placeholders
+        # (thus fixing this problem), at a slight performance cost.
+        fragments = ActiveRecord::Base.connection.unprepared_statement do
+          relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
+        end
+
+        if fragments.any?
+          "(" + fragments.join(")\n#{operator_keyword_fragment}\n(") + ")"
+        else
+          'NULL'
+        end
+      end
+
+      # UNION [ALL] | INTERSECT [ALL] | EXCEPT [ALL]
+      def operator_keyword_fragment
+        remove_duplicates ? self.class.operator_keyword : "#{self.class.operator_keyword} ALL"
+      end
+
+      private
+
+      attr_reader :relations, :remove_duplicates
+    end
+  end
+end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index b15f2ca385a..7fb3487a5e5 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -13,30 +13,9 @@ module Gitlab
     #     sql   = union.to_sql
     #
     #     Project.where("id IN (#{sql})")
-    class Union
-      def initialize(relations, remove_duplicates: true)
-        @relations = relations
-        @remove_duplicates = remove_duplicates
-      end
-
-      def to_sql
-        # Some relations may include placeholders for prepared statements, these
-        # aren't incremented properly when joining relations together this way.
-        # By using "unprepared_statements" we remove the usage of placeholders
-        # (thus fixing this problem), at a slight performance cost.
-        fragments = ActiveRecord::Base.connection.unprepared_statement do
-          @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
-        end
-
-        if fragments.any?
-          "(" + fragments.join(")\n#{union_keyword}\n(") + ")"
-        else
-          'NULL'
-        end
-      end
-
-      def union_keyword
-        @remove_duplicates ? 'UNION' : 'UNION ALL'
+    class Union < SetOperator
+      def self.operator_keyword
+        'UNION'
       end
     end
   end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 504ffe0a2ad..1dd6eab1743 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -426,16 +426,17 @@ module Gitlab
         {} # augmented in EE
       end
 
-      # rubocop: disable CodeReuse/ActiveRecord
       def merge_requests_users(time_period)
-        distinct_count(
-          Event.where(target_type: Event::TARGET_TYPES[:merge_request].to_s).where(time_period),
-          :author_id,
-          start: user_minimum_id,
-          finish: user_maximum_id
-        )
+        counter = Gitlab::UsageDataCounters::TrackUniqueEvents
+
+        redis_usage_data do
+          counter.count_unique_events(
+            event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION,
+            date_from: time_period[:created_at].first,
+            date_to: time_period[:created_at].last
+          )
+        end
       end
-      # rubocop: enable CodeReuse/ActiveRecord
 
       def installation_type
         if Rails.env.production?
diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb
index f2a217e980e..71a4574eea9 100644
--- a/lib/gitlab/usage_data_counters/track_unique_events.rb
+++ b/lib/gitlab/usage_data_counters/track_unique_events.rb
@@ -6,6 +6,7 @@ module Gitlab
       WIKI_ACTION = :wiki_action
       DESIGN_ACTION = :design_action
       PUSH_ACTION = :project_action
+      MERGE_REQUEST_ACTION = :merge_request_action
 
       ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({
         wiki: {
@@ -20,6 +21,12 @@ module Gitlab
         },
         project: {
           pushed: PUSH_ACTION
+        },
+        merge_request: {
+          closed: MERGE_REQUEST_ACTION,
+          merged: MERGE_REQUEST_ACTION,
+          created: MERGE_REQUEST_ACTION,
+          commented: MERGE_REQUEST_ACTION
         }
       }).freeze