Add latest changes from gitlab-org/gitlab@master

4 files changed, 240 insertions, 116 deletions

diff --git a/spec/lib/backup/database_spec.rb b/spec/lib/backup/database_spec.rb
index 95883062fa0..fccd6db0018 100644
--- a/spec/lib/backup/database_spec.rb
+++ b/spec/lib/backup/database_spec.rb
@@ -38,7 +38,7 @@ RSpec.describe Backup::Database do
 
     context 'when the restore command prints errors' do
       let(:visible_error) { "This is a test error\n" }
-      let(:noise) { "Table projects does not exist\n" }
+      let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\n" }
       let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
 
       it 'filters out noise from errors' do
diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb
index 4b7f371b25a..7d26fbb1132 100644
--- a/spec/lib/gitlab/database/migration_helpers_spec.rb
+++ b/spec/lib/gitlab/database/migration_helpers_spec.rb
@@ -2329,4 +2329,56 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
       end
     end
   end
+
+  describe '#create_extension' do
+    subject { model.create_extension(extension) }
+
+    let(:extension) { :btree_gist }
+
+    it 'executes CREATE EXTENSION statement' do
+      expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/)
+
+      subject
+    end
+
+    context 'without proper permissions' do
+      before do
+        allow(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
+      end
+
+      it 'raises the exception' do
+        expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
+      end
+
+      it 'prints an error message' do
+        expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error
+      end
+    end
+  end
+
+  describe '#drop_extension' do
+    subject { model.drop_extension(extension) }
+
+    let(:extension) { 'btree_gist' }
+
+    it 'executes CREATE EXTENSION statement' do
+      expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/)
+
+      subject
+    end
+
+    context 'without proper permissions' do
+      before do
+        allow(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied')
+      end
+
+      it 'raises the exception' do
+        expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/)
+      end
+
+      it 'prints an error message' do
+        expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error
+      end
+    end
+  end
 end
diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
index 036e75ba724..7b9650a966c 100644
--- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
+++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
@@ -8,31 +8,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
   let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' }
   let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' }
 
-  let(:weekly_event) { 'g_analytics_contribution' }
-  let(:daily_event) { 'g_analytics_search' }
-  let(:analytics_slot_event) { 'g_analytics_contribution' }
-  let(:compliance_slot_event) { 'g_compliance_dashboard' }
-  let(:category_analytics) { 'g_analytics_search' }
-  let(:category_productivity) { 'g_analytics_productivity' }
-  let(:no_slot) { 'no_slot' }
-  let(:different_aggregation) { 'different_aggregation' }
-  let(:custom_daily_event) { 'g_analytics_custom' }
-
-  let(:known_events) do
-    [
-      { name: weekly_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "weekly" },
-      { name: daily_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "daily" },
-      { name: category_productivity, redis_slot: "analytics", category: "productivity", aggregation: "weekly" },
-      { name: compliance_slot_event, redis_slot: "compliance", category: "compliance", aggregation: "weekly" },
-      { name: no_slot, category: "global", aggregation: "daily" },
-      { name: different_aggregation, category: "global", aggregation: "monthly" }
-    ].map(&:with_indifferent_access)
-  end
-
-  before do
-    allow(described_class).to receive(:known_events).and_return(known_events)
-  end
-
   around do |example|
     # We need to freeze to a reference time
     # because visits are grouped by the week number in the year
@@ -43,144 +18,223 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
     Timecop.freeze(reference_time) { example.run }
   end
 
-  describe '.events_for_category' do
-    it 'gets the event names for given category' do
-      expect(described_class.events_for_category(:analytics)).to contain_exactly(weekly_event, daily_event)
+  describe '.categories' do
+    it 'gets all unique category names' do
+      expect(described_class.categories).to contain_exactly('analytics', 'compliance', 'ide_edit', 'search')
     end
   end
 
-  describe '.track_event' do
-    it "raise error if metrics don't have same aggregation" do
-      expect { described_class.track_event(entity1, different_aggregation, Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownAggregation)
+  describe 'known_events' do
+    let(:weekly_event) { 'g_analytics_contribution' }
+    let(:daily_event) { 'g_analytics_search' }
+    let(:analytics_slot_event) { 'g_analytics_contribution' }
+    let(:compliance_slot_event) { 'g_compliance_dashboard' }
+    let(:category_analytics_event) { 'g_analytics_search' }
+    let(:category_productivity_event) { 'g_analytics_productivity' }
+    let(:no_slot) { 'no_slot' }
+    let(:different_aggregation) { 'different_aggregation' }
+    let(:custom_daily_event) { 'g_analytics_custom' }
+
+    let(:global_category) { 'global' }
+    let(:compliance_category) {'compliance' }
+    let(:productivity_category) {'productivity' }
+    let(:analytics_category) { 'analytics' }
+
+    let(:known_events) do
+      [
+        { name: weekly_event, redis_slot: "analytics", category: analytics_category, expiry: 84, aggregation: "weekly" },
+        { name: daily_event, redis_slot: "analytics", category: analytics_category, expiry: 84, aggregation: "daily" },
+        { name: category_productivity_event, redis_slot: "analytics", category: productivity_category, aggregation: "weekly" },
+        { name: compliance_slot_event, redis_slot: "compliance", category: compliance_category, aggregation: "weekly" },
+        { name: no_slot, category: global_category, aggregation: "daily" },
+        { name: different_aggregation, category: global_category, aggregation: "monthly" }
+      ].map(&:with_indifferent_access)
+    end
+
+    before do
+      allow(described_class).to receive(:known_events).and_return(known_events)
     end
 
-    it 'raise error if metrics of unknown aggregation' do
-      expect { described_class.track_event(entity1, 'unknown', Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
+    describe '.events_for_category' do
+      it 'gets the event names for given category' do
+        expect(described_class.events_for_category(:analytics)).to contain_exactly(weekly_event, daily_event)
+      end
     end
 
-    context 'for weekly events' do
-      it 'sets the keys in Redis to expire automatically after the given expiry time' do
-        described_class.track_event(entity1, "g_analytics_contribution")
+    describe '.track_event' do
+      it "raise error if metrics don't have same aggregation" do
+        expect { described_class.track_event(entity1, different_aggregation, Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownAggregation)
+      end
 
-        Gitlab::Redis::SharedState.with do |redis|
-          keys = redis.scan_each(match: "g_{analytics}_contribution-*").to_a
-          expect(keys).not_to be_empty
+      it 'raise error if metrics of unknown aggregation' do
+        expect { described_class.track_event(entity1, 'unknown', Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
+      end
 
-          keys.each do |key|
-            expect(redis.ttl(key)).to be_within(5.seconds).of(12.weeks)
+      context 'for weekly events' do
+        it 'sets the keys in Redis to expire automatically after the given expiry time' do
+          described_class.track_event(entity1, "g_analytics_contribution")
+
+          Gitlab::Redis::SharedState.with do |redis|
+            keys = redis.scan_each(match: "g_{analytics}_contribution-*").to_a
+            expect(keys).not_to be_empty
+
+            keys.each do |key|
+              expect(redis.ttl(key)).to be_within(5.seconds).of(12.weeks)
+            end
           end
         end
-      end
 
-      it 'sets the keys in Redis to expire automatically after 6 weeks by default' do
-        described_class.track_event(entity1, "g_compliance_dashboard")
+        it 'sets the keys in Redis to expire automatically after 6 weeks by default' do
+          described_class.track_event(entity1, "g_compliance_dashboard")
 
-        Gitlab::Redis::SharedState.with do |redis|
-          keys = redis.scan_each(match: "g_{compliance}_dashboard-*").to_a
-          expect(keys).not_to be_empty
+          Gitlab::Redis::SharedState.with do |redis|
+            keys = redis.scan_each(match: "g_{compliance}_dashboard-*").to_a
+            expect(keys).not_to be_empty
 
-          keys.each do |key|
-            expect(redis.ttl(key)).to be_within(5.seconds).of(6.weeks)
+            keys.each do |key|
+              expect(redis.ttl(key)).to be_within(5.seconds).of(6.weeks)
+            end
           end
         end
       end
-    end
 
-    context 'for daily events' do
-      it 'sets the keys in Redis to expire after the given expiry time' do
-        described_class.track_event(entity1, "g_analytics_search")
+      context 'for daily events' do
+        it 'sets the keys in Redis to expire after the given expiry time' do
+          described_class.track_event(entity1, "g_analytics_search")
 
-        Gitlab::Redis::SharedState.with do |redis|
-          keys = redis.scan_each(match: "*-g_{analytics}_search").to_a
-          expect(keys).not_to be_empty
+          Gitlab::Redis::SharedState.with do |redis|
+            keys = redis.scan_each(match: "*-g_{analytics}_search").to_a
+            expect(keys).not_to be_empty
 
-          keys.each do |key|
-            expect(redis.ttl(key)).to be_within(5.seconds).of(84.days)
+            keys.each do |key|
+              expect(redis.ttl(key)).to be_within(5.seconds).of(84.days)
+            end
           end
         end
-      end
 
-      it 'sets the keys in Redis to expire after 29 days by default' do
-        described_class.track_event(entity1, "no_slot")
+        it 'sets the keys in Redis to expire after 29 days by default' do
+          described_class.track_event(entity1, "no_slot")
 
-        Gitlab::Redis::SharedState.with do |redis|
-          keys = redis.scan_each(match: "*-{no_slot}").to_a
-          expect(keys).not_to be_empty
+          Gitlab::Redis::SharedState.with do |redis|
+            keys = redis.scan_each(match: "*-{no_slot}").to_a
+            expect(keys).not_to be_empty
 
-          keys.each do |key|
-            expect(redis.ttl(key)).to be_within(5.seconds).of(29.days)
+            keys.each do |key|
+              expect(redis.ttl(key)).to be_within(5.seconds).of(29.days)
+            end
           end
         end
       end
     end
-  end
 
-  describe '.unique_events' do
-    before do
-      # events in current week, should not be counted as week is not complete
-      described_class.track_event(entity1, weekly_event, Date.current)
-      described_class.track_event(entity2, weekly_event, Date.current)
+    describe '.unique_events' do
+      before do
+        # events in current week, should not be counted as week is not complete
+        described_class.track_event(entity1, weekly_event, Date.current)
+        described_class.track_event(entity2, weekly_event, Date.current)
 
-      # Events last week
-      described_class.track_event(entity1, weekly_event, 2.days.ago)
-      described_class.track_event(entity1, weekly_event, 2.days.ago)
-      described_class.track_event(entity1, no_slot, 2.days.ago)
+        # Events last week
+        described_class.track_event(entity1, weekly_event, 2.days.ago)
+        described_class.track_event(entity1, weekly_event, 2.days.ago)
+        described_class.track_event(entity1, no_slot, 2.days.ago)
 
-      # Events 2 weeks ago
-      described_class.track_event(entity1, weekly_event, 2.weeks.ago)
+        # Events 2 weeks ago
+        described_class.track_event(entity1, weekly_event, 2.weeks.ago)
 
-      # Events 4 weeks ago
-      described_class.track_event(entity3, weekly_event, 4.weeks.ago)
-      described_class.track_event(entity4, weekly_event, 29.days.ago)
+        # Events 4 weeks ago
+        described_class.track_event(entity3, weekly_event, 4.weeks.ago)
+        described_class.track_event(entity4, weekly_event, 29.days.ago)
 
-      # events in current day should be counted in daily aggregation
-      described_class.track_event(entity1, daily_event, Date.current)
-      described_class.track_event(entity2, daily_event, Date.current)
+        # events in current day should be counted in daily aggregation
+        described_class.track_event(entity1, daily_event, Date.current)
+        described_class.track_event(entity2, daily_event, Date.current)
 
-      # Events last week
-      described_class.track_event(entity1, daily_event, 2.days.ago)
-      described_class.track_event(entity1, daily_event, 2.days.ago)
+        # Events last week
+        described_class.track_event(entity1, daily_event, 2.days.ago)
+        described_class.track_event(entity1, daily_event, 2.days.ago)
 
-      # Events 2 weeks ago
-      described_class.track_event(entity1, daily_event, 14.days.ago)
+        # Events 2 weeks ago
+        described_class.track_event(entity1, daily_event, 14.days.ago)
 
-      # Events 4 weeks ago
-      described_class.track_event(entity3, daily_event, 28.days.ago)
-      described_class.track_event(entity4, daily_event, 29.days.ago)
-    end
+        # Events 4 weeks ago
+        described_class.track_event(entity3, daily_event, 28.days.ago)
+        described_class.track_event(entity4, daily_event, 29.days.ago)
+      end
 
-    it 'raise error if metrics are not in the same slot' do
-      expect { described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot')
-    end
+      it 'raise error if metrics are not in the same slot' do
+        expect { described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot')
+      end
 
-    it 'raise error if metrics are not in the same category' do
-      expect { described_class.unique_events(event_names: [category_analytics, category_productivity], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
-    end
+      it 'raise error if metrics are not in the same category' do
+        expect { described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
+      end
 
-    it "raise error if metrics don't have same aggregation" do
-      expect { described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level')
-    end
+      it "raise error if metrics don't have same aggregation" do
+        expect { described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level')
+      end
 
-    context 'when data for the last complete week' do
-      it { expect(described_class.unique_events(event_names: weekly_event, start_date: 1.week.ago, end_date: Date.current)).to eq(1) }
-    end
+      context 'when data for the last complete week' do
+        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 1.week.ago, end_date: Date.current)).to eq(1) }
+      end
+
+      context 'when data for the last 4 complete weeks' do
+        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) }
+      end
 
-    context 'when data for the last 4 complete weeks' do
-      it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) }
+      context 'when data for the week 4 weeks ago' do
+        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) }
+      end
+
+      context 'when using daily aggregation' do
+        it { expect(described_class.unique_events(event_names: daily_event, start_date: 7.days.ago, end_date: Date.current)).to eq(2) }
+        it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: Date.current)).to eq(3) }
+        it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) }
+      end
+
+      context 'when no slot is set' do
+        it { expect(described_class.unique_events(event_names: no_slot, start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
+      end
     end
+  end
 
-    context 'when data for the week 4 weeks ago' do
-      it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) }
+  describe 'unique_events_data' do
+    let(:known_events) do
+      [
+        { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+        { name: 'event2_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+        { name: 'event3', category: 'category2', aggregation: "weekly" },
+        { name: 'event4', category: 'category2', aggregation: "weekly" }
+      ].map(&:with_indifferent_access)
     end
 
-    context 'when using daily aggregation' do
-      it { expect(described_class.unique_events(event_names: daily_event, start_date: 7.days.ago, end_date: Date.current)).to eq(2) }
-      it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: Date.current)).to eq(3) }
-      it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) }
+    before do
+      allow(described_class).to receive(:known_events).and_return(known_events)
+      allow(described_class).to receive(:categories).and_return(%w(category1 category2))
+
+      described_class.track_event(entity1, 'event1_slot', 2.days.ago)
+      described_class.track_event(entity2, 'event2_slot', 2.days.ago)
+      described_class.track_event(entity3, 'event2_slot', 2.weeks.ago)
+
+      # events in different slots
+      described_class.track_event(entity2, 'event3', 2.days.ago)
+      described_class.track_event(entity2, 'event4', 2.days.ago)
     end
 
-    context 'when no slot is set' do
-      it { expect(described_class.unique_events(event_names: no_slot, start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
+    it 'returns the number of unique events for all known events' do
+      results = {
+       'category1' => {
+          'event1_slot' => 1,
+          'event2_slot' => 1,
+          'category1_total_unique_counts_weekly' => 2,
+          'category1_total_unique_counts_monthly' => 3
+       },
+       'category2' => {
+          'event3' => 1,
+          'event4' => 1
+       }
+      }
+
+      expect(subject.unique_events_data).to eq(results)
     end
   end
 end
diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb
index a036cb95f63..450d817991a 100644
--- a/spec/lib/gitlab/usage_data_spec.rb
+++ b/spec/lib/gitlab/usage_data_spec.rb
@@ -1141,6 +1141,24 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
     end
   end
 
+  describe 'redis_hll_counters' do
+    subject { described_class.redis_hll_counters }
+
+    let(:categories) { ::Gitlab::UsageDataCounters::HLLRedisCounter.categories }
+
+    it 'has all know_events' do
+      expect(subject).to have_key(:redis_hll_counters)
+
+      expect(subject[:redis_hll_counters].keys).to match_array(categories)
+
+      categories.each do |category|
+        keys = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(category) + ["#{category}_total_unique_counts_weekly", "#{category}_total_unique_counts_monthly"]
+
+        expect(subject[:redis_hll_counters][category].keys).to match_array(keys)
+      end
+    end
+  end
+
   describe '.service_desk_counts' do
     subject { described_class.send(:service_desk_counts) }