Merge branch 'master' into 'catch-redis-address-error'

# Conflicts:
#   lib/gitlab/current_settings.rb

16 files changed, 254 insertions, 68 deletions

diff --git a/config/application.rb b/config/application.rb
index 12242c3b0f5..2f4e2624195 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -26,7 +26,8 @@ module Gitlab
                                      #{config.root}/app/models/members
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
-                                     #{config.root}/app/services/concerns))
+                                     #{config.root}/app/services/concerns
+                                     #{config.root}/app/finders/concerns))
 
     config.generators.templates.push("#{config.root}/generator_templates")
 
@@ -105,11 +106,12 @@ module Gitlab
     config.assets.precompile << "katex.css"
     config.assets.precompile << "katex.js"
     config.assets.precompile << "xterm/xterm.css"
-    config.assets.precompile << "peek.css"
+    config.assets.precompile << "performance_bar.css"
     config.assets.precompile << "lib/ace.js"
     config.assets.precompile << "vendor/assets/fonts/*"
     config.assets.precompile << "test.css"
     config.assets.precompile << "new_nav.css"
+    config.assets.precompile << "new_sidebar.css"
 
     # Version of your assets, change this if you want to expire all your assets
     config.assets.version = '1.0'
@@ -161,5 +163,23 @@ module Gitlab
     config.generators do |g|
       g.factory_girl false
     end
+
+    config.after_initialize do
+      Rails.application.reload_routes!
+
+      project_url_helpers = Module.new do
+        extend ActiveSupport::Concern
+
+        Gitlab::Application.routes.named_routes.helper_names.each do |name|
+          next unless name.include?('namespace_project')
+
+          define_method(name.sub('namespace_project', 'project')) do |project, *args|
+            send(name, project&.namespace, project, *args)
+          end
+        end
+      end
+
+      Gitlab::Routing.add_helpers(project_url_helpers)
+    end
   end
 end
diff --git a/config/boot.rb b/config/boot.rb
index 02baeab29ab..2d01092acd5 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -5,12 +5,6 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
 
 require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
 
-begin
-  require 'bootsnap/setup'
-rescue SystemCallError => exception
-  $stderr.puts "WARNING: Bootsnap failed to setup: #{exception.message}"
-end
-
 # set default directory for multiproces metrics gathering
 if ENV['RAILS_ENV'] == 'development' || ENV['RAILS_ENV'] == 'test'
   ENV['prometheus_multiproc_dir'] ||= 'tmp/prometheus_multiproc_dir'
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 43a8c0078ca..221e3d6e03b 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -450,10 +450,6 @@ production: &base
 
   # Gitaly settings
   gitaly:
-    # This setting controls whether GitLab uses Gitaly (new component
-    # introduced in 9.0). Eventually Gitaly use will become mandatory and
-    # this option will disappear.
-    enabled: true
     # Default Gitaly authentication token. Can be overriden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
@@ -543,6 +539,10 @@ production: &base
     #   enabled: true
     #   host: localhost
     #   port: 3808
+  prometheus:
+    # Time between sampling of unicorn socket metrics, in seconds
+    # unicorn_sampler_interval: 10
+
 
   #
   # 5. Extra customization
@@ -615,6 +615,52 @@ test:
       title: "JIRA"
       url: https://sample_company.atlassian.net
       project_key: PROJECT
+
+  omniauth:
+    enabled: true
+    allow_single_sign_on: true
+    external_providers: []
+
+    providers:
+      - { name: 'cas3',
+          label: 'cas3',
+          args: { url: 'https://sso.example.com',
+                  disable_ssl_verification: false,
+                  login_url: '/cas/login',
+                  service_validate_url: '/cas/p3/serviceValidate',
+                  logout_url: '/cas/logout'} }
+      - { name: 'github',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET',
+          url: "https://github.com/",
+          verify_ssl: false,
+          args: { scope: 'user:email' } }
+      - { name: 'bitbucket',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET' }
+      - { name: 'gitlab',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET',
+          args: { scope: 'api' } }
+      - { name: 'google_oauth2',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET',
+          args: { access_type: 'offline', approval_prompt: '' } }
+      - { name: 'facebook',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET' }
+      - { name: 'twitter',
+          app_id: 'YOUR_APP_ID',
+          app_secret: 'YOUR_APP_SECRET' }
+      - { name: 'auth0',
+          args: {
+            client_id: 'YOUR_AUTH0_CLIENT_ID',
+            client_secret: 'YOUR_AUTH0_CLIENT_SECRET',
+            namespace: 'YOUR_AUTH0_DOMAIN' } }
+      - { name: 'authentiq',
+          app_id: 'YOUR_CLIENT_ID',
+          app_secret: 'YOUR_CLIENT_SECRET',
+          args: { scope: 'aq:name email~rs address aq:push' } }
   ldap:
     enabled: false
     servers:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 8ddf8e4d2e4..fa33e602e93 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -483,7 +483,6 @@ Settings.rack_attack.git_basic_auth['bantime'] ||= 1.hour
 # Gitaly
 #
 Settings['gitaly'] ||= Settingslogic.new({})
-Settings.gitaly['enabled'] = true if Settings.gitaly['enabled'].nil?
 
 #
 # Webpack settings
@@ -495,6 +494,12 @@ Settings.webpack.dev_server['host']    ||= 'localhost'
 Settings.webpack.dev_server['port']    ||= 3808
 
 #
+# Prometheus metrics settings
+#
+Settings['prometheus'] ||= Settingslogic.new({})
+Settings.prometheus['unicorn_sampler_interval'] ||= 10
+
+#
 # Testing settings
 #
 if Rails.env.test?
diff --git a/config/initializers/5_backend.rb b/config/initializers/5_backend.rb
index 2bd159ca7f1..482613dacc9 100644
--- a/config/initializers/5_backend.rb
+++ b/config/initializers/5_backend.rb
@@ -1,6 +1,8 @@
-required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
-current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)
+unless Rails.env.test?
+  required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
+  current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)
 
-unless current_version.valid? && required_version <= current_version
-  warn "WARNING: This version of GitLab depends on gitlab-shell #{required_version}, but you're running #{current_version}. Please update gitlab-shell."
+  unless current_version.valid? && required_version <= current_version
+    warn "WARNING: This version of GitLab depends on gitlab-shell #{required_version}, but you're running #{current_version}. Please update gitlab-shell."
+  end
 end
diff --git a/config/initializers/8_gitaly.rb b/config/initializers/8_gitaly.rb
index 31c7c91d78f..f4f116e67f7 100644
--- a/config/initializers/8_gitaly.rb
+++ b/config/initializers/8_gitaly.rb
@@ -1,8 +1,6 @@
 require 'uri'
 
-if Gitlab.config.gitaly.enabled || Rails.env.test?
-  Gitlab.config.repositories.storages.keys.each do |storage|
-    # Force validation of each address
-    Gitlab::GitalyClient.address(storage)
-  end
+Gitlab.config.repositories.storages.keys.each do |storage|
+  # Force validation of each address
+  Gitlab::GitalyClient.address(storage)
 end
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index a0a63ddf8f0..d56fd7a6cfa 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -119,6 +119,13 @@ def instrument_classes(instrumentation)
 end
 # rubocop:enable Metrics/AbcSize
 
+Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.prometheus.unicorn_sampler_interval).start
+
+Gitlab::Application.configure do |config|
+  # 0 should be Sentry to catch errors in this middleware
+  config.middleware.insert(1, Gitlab::Metrics::ConnectionRackMiddleware)
+end
+
 if Gitlab::Metrics.enabled?
   require 'pathname'
   require 'influxdb'
@@ -175,7 +182,7 @@ if Gitlab::Metrics.enabled?
 
   GC::Profiler.enable
 
-  Gitlab::Metrics::Sampler.new.start
+  Gitlab::Metrics::InfluxSampler.initialize_instance.start
 
   module TrackNewRedisConnections
     def connect(*args)
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index a5636765774..8e2e639fc41 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -87,9 +87,7 @@ Doorkeeper.configure do
   # "password"           => Resource Owner Password Credentials Grant Flow
   # "client_credentials" => Client Credentials Grant Flow
   #
-  # If not specified, Doorkeeper enables all the four grant flows.
-  #
-  grant_flows %w(authorization_code password client_credentials)
+  grant_flows %w(authorization_code implicit password client_credentials)
 
   # Under some circumstances you might want to have applications auto-approved,
   # so that the user skips the authorization step.
diff --git a/config/initializers/doorkeeper_openid_connect.rb b/config/initializers/doorkeeper_openid_connect.rb
index 4ff9019c43c..c58f425b19b 100644
--- a/config/initializers/doorkeeper_openid_connect.rb
+++ b/config/initializers/doorkeeper_openid_connect.rb
@@ -29,7 +29,7 @@ Doorkeeper::OpenidConnect.configure do
       o.claim(:email)          { |user| user.public_email  }
       o.claim(:email_verified) { |user| true if user.public_email? }
       o.claim(:website)        { |user| user.full_website_url if user.website_url? }
-      o.claim(:profile)        { |user| Rails.application.routes.url_helpers.user_url user }
+      o.claim(:profile)        { |user| Gitlab::Routing.url_helpers.user_url user }
       o.claim(:picture)        { |user| user.avatar_url(only_path: false) }
     end
   end
diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb
index 0fee832788d..bfab8c77a4b 100644
--- a/config/initializers/flipper.rb
+++ b/config/initializers/flipper.rb
@@ -1,4 +1,8 @@
 require 'flipper/middleware/memoizer'
 
-Rails.application.config.middleware.use Flipper::Middleware::Memoizer,
-  lambda { Feature.flipper }
+unless Rails.env.test?
+  Rails.application.config.middleware.use Flipper::Middleware::Memoizer,
+    lambda { Feature.flipper }
+
+  Feature.register_feature_groups
+end
diff --git a/config/initializers/relative_naming_ci_namespace.rb b/config/initializers/relative_naming_ci_namespace.rb
index 03ac55be0b6..d9d3034150f 100644
--- a/config/initializers/relative_naming_ci_namespace.rb
+++ b/config/initializers/relative_naming_ci_namespace.rb
@@ -4,10 +4,10 @@
 # - [project.namespace, project, build]
 #
 # instead of:
-# - namespace_project_job_path(project.namespace, project, build)
+# - project_job_path(project, build)
 #
 # Without that, Ci:: namespace is used for resolving routes:
-# - namespace_project_ci_build_path(project.namespace, project, build)
+# - project_ci_build_path(project, build)
 
 module Ci
   def self.use_relative_model_naming?
diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/additional_metrics.yml
index daecde49570..d33fae4182d 100644
--- a/config/prometheus/additional_metrics.yml
+++ b/config/prometheus/additional_metrics.yml
@@ -1,32 +1,82 @@
-- group: Kubernetes
-  priority: 1
+- group: AWS Elastic Load Balancer
+  priority: 10
   metrics:
-  - title: "Memory usage"
-    y_label: "Values"
+  - title: "Throughput"
+    y_label: "Requests / Sec"
     required_metrics:
-      - container_memory_usage_bytes
+      - aws_elb_request_count_sum
     weight: 1
     queries:
-    - query_range: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
-      label: Container memory
-      unit: MiB
-  - title: "Current memory usage"
+    - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) * 60'
+      label: Total
+      unit: req / sec
+  - title: "Latency"
+    y_label: "Latency (ms)"
     required_metrics:
-     - container_memory_usage_bytes
+      - aws_elb_latency_average
     weight: 1
     queries:
-    - query: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
-      display_empty: false
-      unit: MiB
-  - title: "CPU usage"
+    - query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000'
+      label: Average
+      unit: ms
+  - title: "HTTP Error Rate"
+    y_label: "Error Rate (%)"
     required_metrics:
-     - container_cpu_usage_seconds_total
+      - aws_elb_request_count_sum
+      - aws_elb_httpcode_backend_5_xx_sum
+    weight: 1
+    queries:
+    - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
+      label: HTTP Errors
+      unit: "%"
+- group: NGINX
+  priority: 10
+  metrics:
+  - title: "Throughput"
+    y_label: "Requests / Sec"
+    required_metrics:
+      - nginx_requests_total
+    weight: 1
+    queries:
+    - query_range: 'sum(rate(nginx_requests_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m]))'
+      label: Total
+      unit: req / sec
+  - title: "Latency"
+    y_label: "Latency (ms)"
+    required_metrics:
+      - nginx_upstream_response_msecs_avg
+    weight: 1
+    queries:
+    - query_range: 'avg(nginx_upstream_response_msecs_avg{%{environment_filter}}) * 1000'
+      label: Upstream
+      unit: ms
+  - title: "HTTP Error Rate"
+    y_label: "Error Rate (%)"
+    required_metrics:
+      - nginx_responses_total
+    weight: 1
+    queries:
+    - query_range: 'sum(nginx_responses_total{status_code="5xx", %{environment_filter}}) / sum(nginx_responses_total{server_zone!="*", server_zone!="_", %{environment_filter}})'
+      label: HTTP Errors
+      unit: "%"
+- group: Kubernetes
+  priority: 5
+  metrics:
+  - title: "Memory Usage"
+    y_label: "Memory Usage (MB)"
+    required_metrics:
+      - container_memory_usage_bytes
     weight: 1
     queries:
-    - query_range: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'
-  - title: "Current CPU usage"
+    - query_range: '(sum(container_memory_usage_bytes{container_name!="POD",%{environment_filter}}) / count(container_memory_usage_bytes{container_name!="POD",%{environment_filter}})) /1024/1024'
+      label: Average
+      unit: MB
+  - title: "CPU Utilization"
+    y_label: "CPU Utilization (%)"
     required_metrics:
      - container_cpu_usage_seconds_total
     weight: 1
     queries:
-    - query: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'
+    - query_range: 'sum(rate(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}[2m])) / count(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}) * 100'
+      label: Average
+      unit: "%"
diff --git a/config/routes/group.rb b/config/routes/group.rb
index 11cdff55ed8..23052a6c6dc 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -12,7 +12,7 @@ scope(path: 'groups/*group_id',
   end
 
   resource :avatar, only: [:destroy]
-  resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :update, :new, :create] do
+  resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do
     member do
       get :merge_requests
       get :participants
@@ -23,6 +23,14 @@ scope(path: 'groups/*group_id',
   resources :labels, except: [:show] do
     post :toggle_subscription, on: :member
   end
+
+  scope path: '-' do
+    namespace :settings do
+      resource :ci_cd, only: [:show], controller: 'ci_cd'
+    end
+
+    resources :variables, only: [:index, :show, :update, :create, :destroy]
+  end
 end
 
 scope(path: 'groups/*id',
diff --git a/config/routes/legacy_builds.rb b/config/routes/legacy_builds.rb
new file mode 100644
index 00000000000..5ab2b953ce1
--- /dev/null
+++ b/config/routes/legacy_builds.rb
@@ -0,0 +1,22 @@
+resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
+  collection do
+    resources :artifacts, only: [], controller: 'build_artifacts' do
+      collection do
+        get :latest_succeeded,
+          path: '*ref_name_and_path',
+          format: false
+      end
+    end
+  end
+
+  member do
+    get :raw
+  end
+
+  resource :artifacts, only: [], controller: 'build_artifacts' do
+    get :download
+    get :browse, path: 'browse(/*path)', format: false
+    get :file, path: 'file/*path', format: false
+    get :raw, path: 'raw/*path', format: false
+  end
+end
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 19e18c733b1..62cab25c763 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -1,5 +1,4 @@
 require 'constraints/project_url_constrainer'
-require 'gitlab/routes/legacy_builds'
 
 resources :projects, only: [:index, :new, :create]
 
@@ -87,13 +86,8 @@ constraints(ProjectUrlConstrainer.new) do
       resources :forks, only: [:index, :new, :create]
       resource :import, only: [:new, :create, :show]
 
-      resources :merge_requests, concerns: :awardable, constraints: { id: /\d+/ } do
+      resources :merge_requests, concerns: :awardable, except: [:new, :create], constraints: { id: /\d+/ } do
         member do
-          get :commits
-          get :diffs
-          get :conflicts
-          get :conflict_for_path
-          get :pipelines
           get :commit_change_content
           post :merge
           post :cancel_merge_when_pipeline_succeeds
@@ -101,18 +95,32 @@ constraints(ProjectUrlConstrainer.new) do
           get :ci_environments_status
           post :toggle_subscription
           post :remove_wip
-          get :diff_for_path
-          post :resolve_conflicts
           post :assign_related_issues
+
+          scope constraints: { format: nil }, action: :show do
+            get :commits, defaults: { tab: 'commits' }
+            get :pipelines, defaults: { tab: 'pipelines' }
+            get :diffs, defaults: { tab: 'diffs' }
+          end
+
+          scope constraints: { format: 'json' }, as: :json do
+            get :commits
+            get :pipelines
+            get :diffs, to: 'merge_requests/diffs#show'
+          end
+
+          get :diff_for_path, controller: 'merge_requests/diffs'
+
+          scope controller: 'merge_requests/conflicts' do
+            get :conflicts, action: :show
+            get :conflict_for_path
+            post :resolve_conflicts
+          end
         end
 
         collection do
-          get :branch_from
-          get :branch_to
-          get :update_branches
           get :diff_for_path
           post :bulk_update
-          get :new_diffs, path: 'new/diffs'
         end
 
         resources :discussions, only: [], constraints: { id: /\h{40}/ } do
@@ -123,6 +131,29 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
+      controller 'merge_requests/creations', path: 'merge_requests' do
+        post '', action: :create, as: nil
+
+        scope path: 'new', as: :new_merge_request do
+          get '', action: :new
+
+          scope constraints: { format: nil }, action: :new do
+            get :diffs, defaults: { tab: 'diffs' }
+            get :pipelines, defaults: { tab: 'pipelines' }
+          end
+
+          scope constraints: { format: 'json' }, as: :json do
+            get :diffs
+            get :pipelines
+          end
+
+          get :diff_for_path
+          get :update_branches
+          get :branch_from
+          get :branch_to
+        end
+      end
+
       resources :variables, only: [:index, :show, :update, :create, :destroy]
       resources :triggers, only: [:index, :create, :edit, :update, :destroy] do
         member do
@@ -221,7 +252,7 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
-      Gitlab::Routes::LegacyBuilds.new(self).draw
+      draw :legacy_builds
 
       resources :hooks, only: [:index, :create, :edit, :update, :destroy], constraints: { id: /\d+/ } do
         member do
@@ -355,7 +386,7 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
       namespace :settings do
-        resource :members, only: [:show]
+        get :members, to: redirect('/%{namespace_id}/%{project_id}/project_members')
         resource :ci_cd, only: [:show], controller: 'ci_cd'
         resource :integrations, only: [:show]
         resource :repository, only: [:show], controller: :repository
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 2e8c94655c1..c3fdca59a86 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -70,7 +70,8 @@ var config = {
     raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
     test:                 './test.js',
-    peek:                 './peek.js',
+    performance_bar:      './performance_bar.js',
+    webpack_runtime:      './webpack.js',
   },
 
   output: {
@@ -163,6 +164,7 @@ var config = {
         'issue_show',
         'job_details',
         'merge_conflicts',
+        'monitoring',
         'notebook_viewer',
         'pdf_viewer',
         'pipelines',
@@ -189,7 +191,7 @@ var config = {
 
     // create cacheable common library bundles
     new webpack.optimize.CommonsChunkPlugin({
-      names: ['main', 'locale', 'common', 'runtime'],
+      names: ['main', 'locale', 'common', 'webpack_runtime'],
     }),
   ],
 
@@ -244,7 +246,6 @@ if (IS_DEV_SERVER) {
     hot: DEV_SERVER_LIVERELOAD,
     inline: DEV_SERVER_LIVERELOAD
   };
-  config.output.publicPath = '//' + DEV_SERVER_HOST + ':' + DEV_SERVER_PORT + config.output.publicPath;
   config.plugins.push(
     // watch node_modules for changes if we encounter a missing module compile error
     new WatchMissingNodeModulesPlugin(path.join(ROOT_PATH, 'node_modules'))