Add latest changes from gitlab-org/gitlab@master

16 files changed, 348 insertions, 8 deletions

diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 7934b0f8f59..2a6f498269c 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -189,7 +189,7 @@ class MergeRequest < ApplicationRecord
     end
 
     # rubocop: disable CodeReuse/ServiceClass
-    after_transition unchecked: :cannot_be_merged do |merge_request, transition|
+    after_transition [:unchecked, :checking] => :cannot_be_merged do |merge_request, transition|
       if merge_request.notify_conflict?
         NotificationService.new.merge_request_unmergeable(merge_request)
         TodoService.new.merge_request_became_unmergeable(merge_request)
diff --git a/app/services/clusters/applications/prometheus_health_check_service.rb b/app/services/clusters/applications/prometheus_health_check_service.rb
new file mode 100644
index 00000000000..e609d9f0b7b
--- /dev/null
+++ b/app/services/clusters/applications/prometheus_health_check_service.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class PrometheusHealthCheckService
+      include Gitlab::Utils::StrongMemoize
+      include Gitlab::Routing
+
+      def initialize(cluster)
+        @cluster = cluster
+        @logger = Gitlab::AppJsonLogger.build
+      end
+
+      def execute
+        raise 'Invalid cluster type. Only project types are allowed.' unless @cluster.project_type?
+
+        return unless prometheus_application.installed?
+
+        project = @cluster.clusterable
+
+        @logger.info(
+          message: 'Prometheus health check',
+          cluster_id: @cluster.id,
+          newly_unhealthy: became_unhealthy?,
+          currently_healthy: currently_healthy?,
+          was_healthy: was_healthy?
+        )
+
+        send_notification(project) if became_unhealthy?
+
+        prometheus_application.update_columns(healthy: currently_healthy?) if health_changed?
+      end
+
+      private
+
+      def prometheus_application
+        strong_memoize(:prometheus_application) do
+          @cluster.application_prometheus
+        end
+      end
+
+      def currently_healthy?
+        strong_memoize(:currently_healthy) do
+          prometheus_application.prometheus_client.healthy?
+        end
+      end
+
+      def became_unhealthy?
+        strong_memoize(:became_unhealthy) do
+          (was_healthy? || was_healthy?.nil?) && !currently_healthy?
+        end
+      end
+
+      def was_healthy?
+        strong_memoize(:was_healthy) do
+          prometheus_application.healthy
+        end
+      end
+
+      def health_changed?
+        was_healthy? != currently_healthy?
+      end
+
+      def send_notification(project)
+        notification_payload = build_notification_payload(project)
+        token = project.alerts_service.data.token
+        Projects::Alerting::NotifyService.new(project, nil, notification_payload).execute(token)
+        @logger.info(message: 'Successfully notified of Prometheus newly unhealthy', cluster_id: @cluster.id, project_id: project.id)
+      end
+
+      def build_notification_payload(project)
+        cluster_path = namespace_project_cluster_path(
+          project_id: project.path,
+          namespace_id: project.namespace.path,
+          id: @cluster.id
+        )
+
+        {
+          title: "Prometheus is Unhealthy. Cluster Name: #{@cluster.name}",
+          description: "Prometheus is unhealthy for the cluster: [#{@cluster.name}](#{cluster_path}) attached to project #{project.name}."
+        }
+      end
+    end
+  end
+end
diff --git a/changelogs/unreleased/197955-projects-api-improve-api-response-time-for-archived-true.yml b/changelogs/unreleased/197955-projects-api-improve-api-response-time-for-archived-true.yml
new file mode 100644
index 00000000000..65470d467ec
--- /dev/null
+++ b/changelogs/unreleased/197955-projects-api-improve-api-response-time-for-archived-true.yml
@@ -0,0 +1,5 @@
+---
+title: Improve API response for archived project searchs
+merge_request: 27717
+author:
+type: performance
diff --git a/changelogs/unreleased/abrowne-prometheus-healthy-column.yml b/changelogs/unreleased/abrowne-prometheus-healthy-column.yml
new file mode 100644
index 00000000000..0e6b7bb9393
--- /dev/null
+++ b/changelogs/unreleased/abrowne-prometheus-healthy-column.yml
@@ -0,0 +1,5 @@
+---
+title: Add healthy column to clusters_applications_prometheus table
+merge_request: 26168
+author:
+type: added
diff --git a/db/migrate/20200303181648_add_healthy_to_clusters_applications_prometheus.rb b/db/migrate/20200303181648_add_healthy_to_clusters_applications_prometheus.rb
new file mode 100644
index 00000000000..402cdbbdc14
--- /dev/null
+++ b/db/migrate/20200303181648_add_healthy_to_clusters_applications_prometheus.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddHealthyToClustersApplicationsPrometheus < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    # Default is null to indicate that a health check has not run for a project
+    # For now, health checks will only run on monitor demo projects
+    add_column :clusters_applications_prometheus, :healthy, :boolean
+  end
+
+  def down
+    remove_column :clusters_applications_prometheus, :healthy
+  end
+end
diff --git a/db/migrate/20200323134519_add_api_indexes_for_archived_projects.rb b/db/migrate/20200323134519_add_api_indexes_for_archived_projects.rb
new file mode 100644
index 00000000000..cacf427dbe4
--- /dev/null
+++ b/db/migrate/20200323134519_add_api_indexes_for_archived_projects.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AddApiIndexesForArchivedProjects < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  PUBLIC_AND_ARCHIVED_INDEX_NAME = "index_projects_api_created_at_id_for_archived_vis20"
+  ARCHIVED_INDEX_NAME = "index_projects_api_created_at_id_for_archived"
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :projects, [:created_at, :id],
+      where: "archived = true AND visibility_level = 20 AND pending_delete = false",
+      name: PUBLIC_AND_ARCHIVED_INDEX_NAME
+
+    add_concurrent_index :projects, [:created_at, :id], where: "archived = true AND pending_delete = false",
+      name: ARCHIVED_INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :projects, ARCHIVED_INDEX_NAME
+
+    remove_concurrent_index_by_name :projects, PUBLIC_AND_ARCHIVED_INDEX_NAME
+  end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 6301ec6df80..adaa0a594e6 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1748,7 +1748,8 @@ CREATE TABLE public.clusters_applications_prometheus (
     updated_at timestamp with time zone NOT NULL,
     last_update_started_at timestamp with time zone,
     encrypted_alert_manager_token character varying,
-    encrypted_alert_manager_token_iv character varying
+    encrypted_alert_manager_token_iv character varying,
+    healthy boolean
 );
 
 CREATE SEQUENCE public.clusters_applications_prometheus_id_seq
@@ -9654,6 +9655,10 @@ CREATE UNIQUE INDEX index_project_tracing_settings_on_project_id ON public.proje
 
 CREATE INDEX index_projects_api_created_at_id_desc ON public.projects USING btree (created_at, id DESC);
 
+CREATE INDEX index_projects_api_created_at_id_for_archived ON public.projects USING btree (created_at, id) WHERE ((archived = true) AND (pending_delete = false));
+
+CREATE INDEX index_projects_api_created_at_id_for_archived_vis20 ON public.projects USING btree (created_at, id) WHERE ((archived = true) AND (visibility_level = 20) AND (pending_delete = false));
+
 CREATE INDEX index_projects_api_last_activity_at_id_desc ON public.projects USING btree (last_activity_at, id DESC);
 
 CREATE INDEX index_projects_api_name_id_desc ON public.projects USING btree (name, id DESC);
@@ -12731,6 +12736,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200302152516
 20200303055348
 20200303074328
+20200303181648
 20200304085423
 20200304090155
 20200304121828
@@ -12793,6 +12799,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200320123839
 20200323075043
 20200323122201
+20200323134519
 20200324115359
 \.
 
diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md
index 416bd3bfd00..ed03e8a57f3 100644
--- a/doc/user/admin_area/broadcast_messages.md
+++ b/doc/user/admin_area/broadcast_messages.md
@@ -4,14 +4,46 @@ type: reference, howto
 
 # Broadcast Messages **(CORE ONLY)**
 
-GitLab can display messages to all users of a GitLab instance in a banner that appears in the UI.
+GitLab can display broadcast messages to all users of a GitLab instance. There are two types of broadcast messages:
 
-![Broadcast Message](img/broadcast_messages.png)
+- banners
+- notifications
+
+You can style a message's content using the `a` and `br` HTML tags. The `br` tag inserts a line break. The `a` HTML tag accepts `class` and `style` attributes with the following CSS properties:
+
+- `color`
+- `border`
+- `background`
+- `padding`
+- `margin`
+- `text-decoration`
+
+## Banners
+
+Banners are shown on the top of a page.
+
+![Broadcast Message Banner](img/broadcast_messages_banner_v12_10.png)
+
+## Notifications
+
+Notifications are shown on the bottom right of a page and can contain placeholders. A placeholder is replaced with an attribute of the active user. Placeholders must be surrounded by curly braces, for example `{{name}}`.
+The available placeholders are:
+
+- `{{email}}`
+- `{{name}}`
+- `{{user_id}}`
+- `{{username}}`
+- `{{instance_id}}`
+
+If the user is not signed in, user related values will be empty.
+
+![Broadcast Message Notification](img/broadcast_messages_notification_v12_10.png)
 
 Broadcast messages can be managed using the [broadcast messages API](../../api/broadcast_messages.md).
 
 NOTE: **Note:**
 If more than one banner message is active at one time, they are displayed in a stack in order of creation.
+If more than one notification message is active at one time, only the newest is shown.
 
 ## Adding a broadcast message
 
diff --git a/doc/user/admin_area/img/broadcast_messages.png b/doc/user/admin_area/img/broadcast_messages.png
deleted file mode 100644
index f0ae92f8c17..00000000000
--- a/doc/user/admin_area/img/broadcast_messages.png
+++ /dev/null
diff --git a/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png b/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png
new file mode 100644
index 00000000000..f3c468d158e
--- /dev/null
+++ b/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png
diff --git a/doc/user/admin_area/img/broadcast_messages_notification_v12_10.png b/doc/user/admin_area/img/broadcast_messages_notification_v12_10.png
new file mode 100644
index 00000000000..98ea48ccd3c
--- /dev/null
+++ b/doc/user/admin_area/img/broadcast_messages_notification_v12_10.png
diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md
index 58d834bb9d2..a99444c013f 100644
--- a/doc/user/gitlab_com/index.md
+++ b/doc/user/gitlab_com/index.md
@@ -87,6 +87,9 @@ or over the size limit, you can [reduce your repository size with Git](../projec
 | -----------             | ----------------- | ------------- |
 | Repository size including LFS | 10G         | Unlimited     |
 
+NOTE: **Note:**
+A single `git push` is limited to 5GB. LFS is not affected by this limit.
+
 ## IP range
 
 GitLab.com is using the IP range `34.74.90.64/28` for traffic from its Web/API
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 12de20a2c37..71a0d528bd7 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -6,6 +6,7 @@ module Gitlab
     include Gitlab::Utils::StrongMemoize
     Error = Class.new(StandardError)
     QueryError = Class.new(Gitlab::PrometheusClient::Error)
+    HEALTHY_RESPONSE = "Prometheus is Healthy.\n"
 
     # Target number of data points for `query_range`.
     # Please don't exceed the limit of 11000 data points
@@ -32,13 +33,20 @@ module Gitlab
       json_api_get('query', query: '1')
     end
 
+    def healthy?
+      response_body = handle_management_api_response(get(health_url, {}))
+
+      # From Prometheus docs: This endpoint always returns 200 and should be used to check Prometheus health.
+      response_body == HEALTHY_RESPONSE
+    end
+
     def proxy(type, args)
       path = api_path(type)
       get(path, args)
     rescue Gitlab::HTTP::ResponseError => ex
       raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code)
 
-      handle_response(ex.response)
+      handle_querying_api_response(ex.response)
     end
 
     def query(query, time: Time.now)
@@ -79,6 +87,10 @@ module Gitlab
       [QUERY_RANGE_MIN_STEP, step].max
     end
 
+    def health_url
+      [api_url, '-/healthy'].join('/')
+    end
+
     private
 
     def api_path(type)
@@ -88,11 +100,11 @@ module Gitlab
     def json_api_get(type, args = {})
       path = api_path(type)
       response = get(path, args)
-      handle_response(response)
+      handle_querying_api_response(response)
     rescue Gitlab::HTTP::ResponseError => ex
       raise PrometheusClient::Error, "Network connection error" unless ex.response && ex.response.try(:code)
 
-      handle_response(ex.response)
+      handle_querying_api_response(ex.response)
     end
 
     def gitlab_http_key(key)
@@ -119,7 +131,15 @@ module Gitlab
       raise PrometheusClient::Error, 'Connection refused'
     end
 
-    def handle_response(response)
+    def handle_management_api_response(response)
+      if response.code == 200
+        response.body
+      else
+        raise PrometheusClient::Error, "#{response.code} - #{response.body}"
+      end
+    end
+
+    def handle_querying_api_response(response)
       response_code = response.try(:code)
       response_body = response.try(:body)
 
diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb
index 5eb133a5bf4..e869a384b29 100644
--- a/spec/lib/gitlab/prometheus_client_spec.rb
+++ b/spec/lib/gitlab/prometheus_client_spec.rb
@@ -16,6 +16,26 @@ describe Gitlab::PrometheusClient do
     end
   end
 
+  describe '#healthy?' do
+    it 'returns true when status code is 200 and healthy response body' do
+      stub_request(:get, subject.health_url).to_return(status: 200, body: described_class::HEALTHY_RESPONSE)
+
+      expect(subject.healthy?).to eq(true)
+    end
+
+    it 'returns false when status code is 200 and unhealthy response body' do
+      stub_request(:get, subject.health_url).to_return(status: 200, body: '')
+
+      expect(subject.healthy?).to eq(false)
+    end
+
+    it 'raises error when status code not 200' do
+      stub_request(:get, subject.health_url).to_return(status: 500, body: '')
+
+      expect { subject.healthy? }.to raise_error(Gitlab::PrometheusClient::Error)
+    end
+  end
+
   # This shared examples expect:
   # - query_url: A query URL
   # - execute_query: A query call
diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb
index 137795dcbc3..f37e39a9ee9 100644
--- a/spec/models/merge_request_spec.rb
+++ b/spec/models/merge_request_spec.rb
@@ -3223,6 +3223,14 @@ describe MergeRequest do
             subject.mark_as_unmergeable
           end
 
+          it 'notifies conflict, with enabled async mergability check' do
+            expect(notification_service).to receive(:merge_request_unmergeable).with(subject).once
+            expect(todo_service).to receive(:merge_request_became_unmergeable).with(subject).once
+
+            subject.mark_as_checking
+            subject.mark_as_unmergeable
+          end
+
           it 'does not notify whenever merge request is newly unmergeable due to other reasons' do
             allow(subject.project.repository).to receive(:can_be_merged?).and_return(true)
 
diff --git a/spec/services/clusters/applications/prometheus_health_check_service_spec.rb b/spec/services/clusters/applications/prometheus_health_check_service_spec.rb
new file mode 100644
index 00000000000..5c4127e4938
--- /dev/null
+++ b/spec/services/clusters/applications/prometheus_health_check_service_spec.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Applications::PrometheusHealthCheckService, '#execute' do
+  let(:service) { described_class.new(cluster) }
+
+  subject { service.execute }
+
+  RSpec.shared_examples 'no alert' do
+    it 'does not send alert' do
+      expect(Projects::Alerting::NotifyService).not_to receive(:new)
+
+      subject
+    end
+  end
+
+  RSpec.shared_examples 'sends alert' do
+    it 'sends an alert' do
+      expect_next_instance_of(Projects::Alerting::NotifyService) do |notify_service|
+        expect(notify_service).to receive(:execute).with(alerts_service.token)
+      end
+
+      subject
+    end
+  end
+
+  RSpec.shared_examples 'correct health stored' do
+    it 'stores the correct health of prometheus app' do
+      subject
+
+      expect(prometheus.healthy).to eq(client_healthy)
+    end
+  end
+
+  context 'when cluster is not project_type' do
+    let(:cluster) { create(:cluster, :instance) }
+
+    it { expect { subject }.to raise_error(RuntimeError, 'Invalid cluster type. Only project types are allowed.') }
+  end
+
+  context 'when cluster is project_type' do
+    let_it_be(:alerts_service) { create(:alerts_service) }
+    let_it_be(:project) { create(:project, alerts_service: alerts_service) }
+    let(:applications_prometheus_healthy) { true }
+    let(:prometheus) { create(:clusters_applications_prometheus, status: prometheus_status_value, healthy: applications_prometheus_healthy) }
+    let(:cluster) { create(:cluster, :project, application_prometheus: prometheus, projects: [project]) }
+
+    context 'when prometheus not installed' do
+      let(:prometheus_status_value) { Clusters::Applications::Prometheus.state_machine.states[:installing].value }
+
+      it { expect(subject).to eq(nil) }
+      include_examples 'no alert'
+    end
+
+    context 'when prometheus installed' do
+      let(:prometheus_status_value) { Clusters::Applications::Prometheus.state_machine.states[:installed].value }
+
+      before do
+        client = instance_double('PrometheusClient', healthy?: client_healthy)
+        expect(prometheus).to receive(:prometheus_client).and_return(client)
+      end
+
+      context 'when newly unhealthy' do
+        let(:applications_prometheus_healthy) { true }
+        let(:client_healthy) { false }
+
+        include_examples 'sends alert'
+        include_examples 'correct health stored'
+      end
+
+      context 'when newly healthy' do
+        let(:applications_prometheus_healthy) { false }
+        let(:client_healthy) { true }
+
+        include_examples 'no alert'
+        include_examples 'correct health stored'
+      end
+
+      context 'when continuously unhealthy' do
+        let(:applications_prometheus_healthy) { false }
+        let(:client_healthy) { false }
+
+        include_examples 'no alert'
+        include_examples 'correct health stored'
+      end
+
+      context 'when continuously healthy' do
+        let(:applications_prometheus_healthy) { true }
+        let(:client_healthy) { true }
+
+        include_examples 'no alert'
+        include_examples 'correct health stored'
+      end
+
+      context 'when first health check and healthy' do
+        let(:applications_prometheus_healthy) { nil }
+        let(:client_healthy) { true }
+
+        include_examples 'no alert'
+        include_examples 'correct health stored'
+      end
+
+      context 'when first health check and not healthy' do
+        let(:applications_prometheus_healthy) { nil }
+        let(:client_healthy) { false }
+
+        include_examples 'sends alert'
+        include_examples 'correct health stored'
+      end
+    end
+  end
+end