Add latest changes from gitlab-org/gitlab@13-4-stable-ee

3 files changed, 75 insertions, 4 deletions

diff --git a/lib/gitlab/utils/gzip.rb b/lib/gitlab/utils/gzip.rb
new file mode 100644
index 00000000000..898be651554
--- /dev/null
+++ b/lib/gitlab/utils/gzip.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Utils
+    module Gzip
+      def gzip_compress(data)
+        # .compress returns ASCII-8BIT, so we need to force the encoding to
+        #   UTF-8 before caching it in redis, else we risk encoding mismatch
+        #   errors.
+        #
+        ActiveSupport::Gzip.compress(data).force_encoding("UTF-8")
+      rescue Zlib::GzipFile::Error
+        data
+      end
+
+      def gzip_decompress(data)
+        # Since we could be dealing with an already populated cache full of data
+        #   that isn't gzipped, we want to also check to see if the data is
+        #   gzipped before we attempt to .decompress it, thus we check the first
+        #   2 bytes for "\x1F\x8B" to confirm it is a gzipped string. While a
+        #   non-gzipped string will raise a Zlib::GzipFile::Error, which we're
+        #   rescuing, we don't want to count on rescue for control flow.
+        #
+        data[0..1] == "\x1F\x8B" ? ActiveSupport::Gzip.decompress(data) : data
+      rescue Zlib::GzipFile::Error
+        data
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb
index 82c4a0e3b23..e783ac785cc 100644
--- a/lib/gitlab/utils/markdown.rb
+++ b/lib/gitlab/utils/markdown.rb
@@ -4,11 +4,13 @@ module Gitlab
   module Utils
     module Markdown
       PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze
+      PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate)(\s+only)?\)\**/.freeze
 
       def string_to_anchor(string)
         string
           .strip
           .downcase
+          .gsub(PRODUCT_SUFFIX, '')
           .gsub(PUNCTUATION_REGEXP, '') # remove punctuation
           .tr(' ', '-') # replace spaces with dash
           .squeeze('-') # replace multiple dashes with one
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index 36046ca14bf..ca6a36c9cea 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -59,6 +59,12 @@ module Gitlab
         FALLBACK
       end
 
+      def sum(relation, column, batch_size: nil, start: nil, finish: nil)
+        Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish)
+      rescue ActiveRecord::StatementInvalid
+        FALLBACK
+      end
+
       def alt_usage_data(value = nil, fallback: FALLBACK, &block)
         if block_given?
           yield
@@ -77,11 +83,11 @@ module Gitlab
         end
       end
 
-      def with_prometheus_client(fallback: nil)
-        return fallback unless Gitlab::Prometheus::Internal.prometheus_enabled?
+      def with_prometheus_client(fallback: nil, verify: true)
+        client = prometheus_client(verify: verify)
+        return fallback unless client
 
-        prometheus_address = Gitlab::Prometheus::Internal.uri
-        yield Gitlab::PrometheusClient.new(prometheus_address, allow_local_requests: true)
+        yield client
       end
 
       def measure_duration
@@ -96,8 +102,41 @@ module Gitlab
         yield.merge(key => Time.current)
       end
 
+      # @param event_name [String] the event name
+      # @param values [Array|String] the values counted
+      def track_usage_event(event_name, values)
+        return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true)
+        return unless Gitlab::CurrentSettings.usage_ping_enabled?
+
+        Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s)
+      end
+
       private
 
+      def prometheus_client(verify:)
+        server_address = prometheus_server_address
+
+        return unless server_address
+
+        # There really is not a way to discover whether a Prometheus connection is using TLS or not
+        # Try TLS first because HTTPS will return fast if failed.
+        %w[https http].find do |scheme|
+          api_url = "#{scheme}://#{server_address}"
+          client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify)
+          break client if client.ready?
+        rescue
+          nil
+        end
+      end
+
+      def prometheus_server_address
+        if Gitlab::Prometheus::Internal.prometheus_enabled?
+          Gitlab::Prometheus::Internal.server_address
+        elsif Gitlab::Consul::Internal.api_url
+          Gitlab::Consul::Internal.discover_prometheus_server_address
+        end
+      end
+
       def redis_usage_counter
         yield
       rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent