Merge branch 'master' into zj-project-templates

* master: (623 commits)
  Fix issues with pdf-js dependencies
  fix missing changelog entries for security release on 2017-01-23
  Update top bar issues icon
  Fix pipeline icon in contextual nav for projects
  Since mysql is not a priority anymore, test it less
  Fix order of CI lint ace editor loading
  Add container registry and spam logs icons
  Fix different Markdown styles
  Backport to CE for:
  Make new dropdown dividers full width
  Fix spec
  Fix spec
  Fix spec
  Bump GITLAB_SHELL_VERSION and GITALY_VERSION to support unhiding refs
  Add changelog
  Install yarn via apt in update guides
  Use long curl options
  fix
  Add a spec for concurrent process
  Remove monkey-patched Array.prototype.first() and last() methods
  ...

18 files changed, 178 insertions, 40 deletions

diff --git a/config/application.rb b/config/application.rb
index 1c13cc81270..f7145566262 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -23,13 +23,13 @@ module Gitlab
     # https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687
     # This is a nice reference article on autoloading/eager loading:
     # http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
-    config.eager_load_paths.push(*%W(#{config.root}/lib
+    config.eager_load_paths.push(*%W[#{config.root}/lib
                                      #{config.root}/app/models/hooks
                                      #{config.root}/app/models/members
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
                                      #{config.root}/app/services/concerns
-                                     #{config.root}/app/finders/concerns))
+                                     #{config.root}/app/finders/concerns])
 
     config.generators.templates.push("#{config.root}/generator_templates")
 
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index cb007813b65..73a68c6da1b 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -228,7 +228,8 @@ production: &base
   # ==========================
 
   ## LDAP settings
-  # You can inspect a sample of the LDAP users with login access by running:
+  # You can test connections and inspect a sample of the LDAP users with login
+  # access by running:
   #   bundle exec rake gitlab:ldap:check RAILS_ENV=production
   ldap:
     enabled: false
@@ -251,13 +252,45 @@ production: &base
         # Example: 'Paris' or 'Acme, Ltd.'
         label: 'LDAP'
 
+        # Example: 'ldap.mydomain.com'
         host: '_your_ldap_server'
-        port: 389
-        uid: 'sAMAccountName'
-        method: 'plain' # "tls" or "ssl" or "plain"
+        # This port is an example, it is sometimes different but it is always an integer and not a string
+        port: 389 # usually 636 for SSL
+        uid: 'sAMAccountName' # This should be the attribute, not the value that maps to uid.
+
+        # Examples: 'america\\momo' or 'CN=Gitlab Git,CN=Users,DC=mydomain,DC=com'
         bind_dn: '_the_full_dn_of_the_user_you_will_bind_with'
         password: '_the_password_of_the_bind_user'
 
+        # Encryption method. The "method" key is deprecated in favor of
+        # "encryption".
+        #
+        #   Examples: "start_tls" or "simple_tls" or "plain"
+        #
+        #   Deprecated values: "tls" was replaced with "start_tls" and "ssl" was
+        #   replaced with "simple_tls".
+        #
+        encryption: 'plain'
+
+        # Enables SSL certificate verification if encryption method is
+        # "start_tls" or "simple_tls". (Defaults to false for backward-
+        # compatibility)
+        verify_certificates: false
+
+        # Specifies the path to a file containing a PEM-format CA certificate,
+        # e.g. if you need to use an internal CA.
+        #
+        #   Example: '/etc/ca.pem'
+        #
+        ca_file: ''
+
+        # Specifies the SSL version for OpenSSL to use, if the OpenSSL default
+        # is not appropriate.
+        #
+        #   Example: 'TLSv1_1'
+        #
+        ssl_version: ''
+
         # Set a timeout, in seconds, for LDAP queries. This helps avoid blocking
         # a request if the LDAP server becomes unresponsive.
         # A value of 0 means there is no timeout.
@@ -286,17 +319,20 @@ production: &base
 
         # Base where we can search for users
         #
-        #   Ex. ou=People,dc=gitlab,dc=example
+        #   Ex. 'ou=People,dc=gitlab,dc=example' or 'DC=mydomain,DC=com'
         #
         base: ''
 
         # Filter LDAP users
         #
-        #   Format: RFC 4515 http://tools.ietf.org/search/rfc4515
+        #   Format: RFC 4515 https://tools.ietf.org/search/rfc4515
         #   Ex. (employeeType=developer)
         #
         #   Note: GitLab does not support omniauth-ldap's custom filter syntax.
         #
+        #   Example for getting only specific users:
+        #   '(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'
+        #
         user_filter: ''
 
         # LDAP attributes that GitLab will use to create an account for the LDAP user.
@@ -674,7 +710,7 @@ test:
         host: 127.0.0.1
         port: 3890
         uid: 'uid'
-        method: 'plain' # "tls" or "ssl" or "plain"
+        encryption: 'plain' # "start_tls" or "simple_tls" or "plain"
         base: 'dc=example,dc=com'
         user_filter: ''
         group_base: 'ou=groups,dc=example,dc=com'
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index ec7ce51b542..63f4c8c9e0a 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -145,6 +145,24 @@ if Settings.ldap['enabled'] || Rails.env.test?
     server['attributes'] = {} if server['attributes'].nil?
     server['provider_name'] ||= "ldap#{key}".downcase
     server['provider_class'] = OmniAuth::Utils.camelize(server['provider_name'])
+
+    # For backwards compatibility
+    server['encryption'] ||= server['method']
+    server['encryption'] = 'simple_tls' if server['encryption'] == 'ssl'
+    server['encryption'] = 'start_tls' if server['encryption'] == 'tls'
+
+    # Certificates are not verified for backwards compatibility.
+    # This default should be flipped to true in 9.5.
+    if server['verify_certificates'].nil?
+      server['verify_certificates'] = false
+
+      message = <<-MSG.strip_heredoc
+        LDAP SSL certificate verification is disabled for backwards-compatibility.
+        Please add the "verify_certificates" option to gitlab.yml for each LDAP
+        server. Certificate verification will be enabled by default in GitLab 9.5.
+      MSG
+      Rails.logger.warn(message)
+    end
   end
 end
 
@@ -205,7 +223,7 @@ Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_c
 Settings.gitlab['host']       ||= ENV['GITLAB_HOST'] || 'localhost'
 Settings.gitlab['ssh_host']   ||= Settings.gitlab.host
 Settings.gitlab['https']        = false if Settings.gitlab['https'].nil?
-Settings.gitlab['port']       ||= Settings.gitlab.https ? 443 : 80
+Settings.gitlab['port']       ||= ENV['GITLAB_PORT'] || (Settings.gitlab.https ? 443 : 80)
 Settings.gitlab['relative_url_root'] ||= ENV['RAILS_RELATIVE_URL_ROOT'] || ''
 Settings.gitlab['protocol'] ||= Settings.gitlab.https ? "https" : "http"
 Settings.gitlab['email_enabled'] ||= true if Settings.gitlab['email_enabled'].nil?
@@ -441,10 +459,6 @@ Settings.backup['pg_schema']    = nil
 Settings.backup['path']         = Settings.absolute(Settings.backup['path'] || "tmp/backups/")
 Settings.backup['archive_permissions'] ||= 0600
 Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil })
-# Convert upload connection settings to use symbol keys, to make Fog happy
-if Settings.backup['upload']['connection']
-  Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
-end
 Settings.backup['upload']['multipart_chunk_size'] ||= 104857600
 Settings.backup['upload']['encryption'] ||= nil
 Settings.backup['upload']['storage_class'] ||= nil
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index 987324a86c9..a2f8421f5d7 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -6,7 +6,7 @@ Prometheus::Client.configure do |config|
   config.initial_mmap_file_size = 4 * 1024
   config.multiprocess_files_dir = ENV['prometheus_multiproc_dir']
 
-  if Rails.env.development? && Rails.env.test?
+  if Rails.env.development? || Rails.env.test?
     config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
   end
 end
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index 25630b298ce..2aeb94d47cd 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -114,6 +114,9 @@ def instrument_classes(instrumentation)
   # This is a Rails scope so we have to instrument it manually.
   instrumentation.instrument_method(Project, :visible_to_user)
 
+  # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/34509
+  instrumentation.instrument_method(MarkupHelper, :link_to_gfm)
+
   # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159
   instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits)
 end
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 3b1317030bc..3aed2136f1b 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -192,7 +192,7 @@ Devise.setup do |config|
   #
   # The :"*/*" and "*/*" formats below is required to match Internet
   # Explorer requests.
-  # config.navigational_formats = [:"*/*", "*/*", :html]
+  config.navigational_formats = [:"*/*", "*/*", :html, :zip]
 
   # The default HTTP method used to sign out a resource. Default is :delete.
   config.sign_out_via = :delete
@@ -206,11 +206,11 @@ Devise.setup do |config|
   # If you want to use other strategies, that are not supported by Devise, or
   # change the failure app, you can configure them inside the config.warden block.
   #
-  config.warden do |manager|
-    manager.failure_app = Gitlab::DeviseFailure
-    # manager.intercept_401 = false
-    # manager.default_strategies(scope: :user).unshift :some_external_strategy
-  end
+  # config.warden do |manager|
+  #   manager.failure_app = Gitlab::DeviseFailure
+  #   manager.intercept_401 = false
+  #   manager.default_strategies(scope: :user).unshift :some_external_strategy
+  # end
 
   if Gitlab::LDAP::Config.enabled?
     Gitlab::LDAP::Config.providers.each do |provider|
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 8e2e639fc41..40e635bf2cf 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -92,9 +92,9 @@ Doorkeeper.configure do
   # Under some circumstances you might want to have applications auto-approved,
   # so that the user skips the authorization step.
   # For example if dealing with trusted a application.
-  # skip_authorization do |resource_owner, client|
-  #   client.superapp? or resource_owner.admin?
-  # end
+  skip_authorization do |resource_owner, client|
+    client.application.trusted?
+  end
 
   # WWW-Authenticate Realm (default "Doorkeeper").
   # realm "Doorkeeper"
diff --git a/config/initializers/grape_route_helpers_fix.rb b/config/initializers/grape_route_helpers_fix.rb
new file mode 100644
index 00000000000..d3cf9e453d0
--- /dev/null
+++ b/config/initializers/grape_route_helpers_fix.rb
@@ -0,0 +1,35 @@
+if defined?(GrapeRouteHelpers)
+  module GrapeRouteHelpers
+    class DecoratedRoute
+      # GrapeRouteHelpers gem tries to parse the versions
+      # from a string, not supporting Grape `version` array definition.
+      #
+      # Without the following fix, we get this on route helpers generation:
+      #
+      # => undefined method `scan' for ["v3", "v4"]
+      #
+      # 2.0.0 implementation of this method:
+      #
+      # ```
+      # def route_versions
+      #   version_pattern = /[^\[",\]\s]+/
+      #   if route_version
+      #     route_version.scan(version_pattern)
+      #   else
+      #     [nil]
+      #   end
+      # end
+      # ```
+      def route_versions
+        return [nil] if route_version.nil? || route_version.empty?
+
+        if route_version.is_a?(String)
+          version_pattern = /[^\[",\]\s]+/
+          route_version.scan(version_pattern)
+        else
+          route_version
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb
index c9a515dfcd5..21fe8d72459 100644
--- a/config/initializers/lograge.rb
+++ b/config/initializers/lograge.rb
@@ -14,7 +14,10 @@ unless Sidekiq.server?
     config.lograge.custom_options = lambda do |event|
       {
         time: event.time.utc.iso8601(3),
-        params: event.payload[:params].except(%w(controller action format))
+        params: event.payload[:params].except(*%w(controller action format)),
+        remote_ip: event.payload[:remote_ip],
+        user_id: event.payload[:user_id],
+        username: event.payload[:username]
       }
     end
   end
diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb
new file mode 100644
index 00000000000..de0bc5322aa
--- /dev/null
+++ b/config/initializers/mysql_set_length_for_binary_indexes.rb
@@ -0,0 +1,21 @@
+# This patches ActiveRecord so indexes for binary columns created using the
+# MySQL adapter apply a length of 20. Otherwise MySQL can't create an index on
+# binary columns.
+
+module MysqlSetLengthForBinaryIndex
+  def add_index(table_name, column_names, options = {})
+    Array(column_names).each do |column_name|
+      column = ActiveRecord::Base.connection.columns(table_name).find { |c| c.name == column_name }
+
+      if column&.type == :binary
+        options[:length] = 20
+      end
+    end
+
+    super(table_name, column_names, options)
+  end
+end
+
+if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+  ActiveRecord::ConnectionAdapters::Mysql2Adapter.send(:prepend, MysqlSetLengthForBinaryIndex)
+end
diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb
index f7fa6d1c2de..56c279ffcf4 100644
--- a/config/initializers/omniauth.rb
+++ b/config/initializers/omniauth.rb
@@ -1,13 +1,16 @@
 if Gitlab::LDAP::Config.enabled?
   module OmniAuth::Strategies
-    server = Gitlab.config.ldap.servers.values.first
-    klass = server['provider_class']
-    const_set(klass, Class.new(LDAP)) unless klass == 'LDAP'
+    Gitlab::LDAP::Config.available_servers.each do |server|
+      # do not redeclare LDAP
+      next if server['provider_name'] == 'ldap'
+      const_set(server['provider_class'], Class.new(LDAP))
+    end
   end
 
   OmniauthCallbacksController.class_eval do
-    server = Gitlab.config.ldap.servers.values.first
-    alias_method server['provider_name'], :ldap
+    Gitlab::LDAP::Config.available_servers.each do |server|
+      alias_method server['provider_name'], :ldap
+    end
   end
 end
 
@@ -16,7 +19,7 @@ OmniAuth.config.allowed_request_methods = [:post]
 # In case of auto sign-in, the GET method is used (users don't get to click on a button)
 OmniAuth.config.allowed_request_methods << :get if Gitlab.config.omniauth.auto_sign_in_with_provider.present?
 OmniAuth.config.before_request_phase do |env|
-  OmniAuth::RequestForgeryProtection.call(env)
+  Gitlab::RequestForgeryProtection.call(env)
 end
 
 if Gitlab.config.omniauth.enabled
diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/additional_metrics.yml
index 61d39e7bfcf..5eb01d62924 100644
--- a/config/prometheus/additional_metrics.yml
+++ b/config/prometheus/additional_metrics.yml
@@ -1,4 +1,4 @@
-- group: HA Proxy
+- group: Response metrics (HA Proxy)
   priority: 10
   metrics:
   - title: "Throughput"
@@ -19,7 +19,7 @@
     - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))'
       label: HTTP Errors
       unit: "%"
-- group: AWS Elastic Load Balancer
+- group: Response metrics (AWS ELB)
   priority: 10
   metrics:
   - title: "Throughput"
@@ -50,7 +50,7 @@
     - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
       label: HTTP Errors
       unit: "%"
-- group: NGINX
+- group: Response metrics (NGINX)
   priority: 10
   metrics:
   - title: "Throughput"
@@ -80,7 +80,7 @@
     - query_range: 'sum(rate(nginx_responses_total{status_code="5xx", %{environment_filter}}[2m])) / sum(rate(nginx_requests_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m]))'
       label: HTTP Errors
       unit: "%"
-- group: Kubernetes
+- group: System metrics (Kubernetes)
   priority: 5
   metrics:
   - title: "Memory Usage"
diff --git a/config/routes/api.rb b/config/routes/api.rb
index 69c8efc151c..ce7a7c88900 100644
--- a/config/routes/api.rb
+++ b/config/routes/api.rb
@@ -1,2 +1,2 @@
 API::API.logger Rails.logger
-mount API::API => '/api'
+mount API::API => '/'
diff --git a/config/routes/profile.rb b/config/routes/profile.rb
index 3dc890e5785..3e4e6111ab8 100644
--- a/config/routes/profile.rb
+++ b/config/routes/profile.rb
@@ -23,6 +23,11 @@ resource :profile, only: [:show, :update] do
     end
     resource :preferences, only: [:show, :update]
     resources :keys, only: [:index, :show, :create, :destroy]
+    resources :gpg_keys, only: [:index, :create, :destroy] do
+      member do
+        put :revoke
+      end
+    end
     resources :emails, only: [:index, :create, :destroy]
     resources :chat_names, only: [:index, :new, :create, :destroy] do
       collection do
diff --git a/config/routes/repository.rb b/config/routes/repository.rb
index 11911636fa7..2ba16035ece 100644
--- a/config/routes/repository.rb
+++ b/config/routes/repository.rb
@@ -2,7 +2,7 @@
 
 resource :repository, only: [:create] do
   member do
-    get 'archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex }
+    get ':ref/archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex, ref: /.+/ }, action: 'archive', as: 'archive'
   end
 end
 
@@ -76,6 +76,8 @@ scope format: false do
     get '/tree/*id', to: 'tree#show', as: :tree
     get '/raw/*id', to: 'raw#show', as: :raw
     get '/blame/*id', to: 'blame#show', as: :blame
+
+    get '/commits/*id/signatures', to: 'commits#signatures', as: :signatures
     get '/commits/*id', to: 'commits#show', as: :commits
 
     post '/create_dir/*id', to: 'tree#create_dir', as: :create_dir
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 1d9e69a2408..7496bfa4fbb 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -29,6 +29,8 @@
   - [email_receiver, 2]
   - [emails_on_push, 2]
   - [mailers, 2]
+  - [invalid_gpg_signature_update, 2]
+  - [create_gpg_signature, 2]
   - [upload_checksum, 1]
   - [use_key, 1]
   - [repository_fork, 1]
diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example
index 40a16a32359..cc10da2bd88 100644
--- a/config/unicorn.rb.example
+++ b/config/unicorn.rb.example
@@ -121,6 +121,10 @@ after_fork do |server, worker|
   defined?(ActiveRecord::Base) and
     ActiveRecord::Base.establish_connection
 
+  # reset prometheus client, this will cause any opened metrics files to be closed
+  defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
+    Prometheus::Client.reinitialize_on_pid_change
+
   # if preload_app is true, then you may also want to check and
   # restart any other shared sockets/descriptors such as Memcached,
   # and Redis.  TokyoCabinet file handles are safe to reuse
diff --git a/config/webpack.config.js b/config/webpack.config.js
index f08daa2fddb..32e57c43bd9 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -39,6 +39,8 @@ var config = {
     environments_folder:  './environments/folder/environments_folder_bundle.js',
     filtered_search:      './filtered_search/filtered_search_bundle.js',
     graphs:               './graphs/graphs_bundle.js',
+    graphs_charts:        './graphs/graphs_charts.js',
+    graphs_show:          './graphs/graphs_show.js',
     group:                './group.js',
     groups:               './groups/index.js',
     groups_list:          './groups_list.js',
@@ -54,8 +56,12 @@ var config = {
     notebook_viewer:      './blob/notebook_viewer.js',
     pdf_viewer:           './blob/pdf_viewer.js',
     pipelines:            './pipelines/pipelines_bundle.js',
-    pipelines_details:     './pipelines/pipeline_details_bundle.js',
+    pipelines_charts:     './pipelines/pipelines_charts.js',
+    pipelines_details:    './pipelines/pipeline_details_bundle.js',
+    pipelines_times:      './pipelines/pipelines_times.js',
     profile:              './profile/profile_bundle.js',
+    project_import_gl:    './projects/project_import_gitlab_project.js',
+    project_new:          './projects/project_new.js',
     prometheus_metrics:   './prometheus_metrics',
     protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
@@ -67,10 +73,12 @@ var config = {
     stl_viewer:           './blob/stl_viewer.js',
     terminal:             './terminal/terminal_bundle.js',
     u2f:                  ['vendor/u2f'],
+    ui_development_kit:   './ui_development_kit.js',
     users:                './users/index.js',
     raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
     test:                 './test.js',
+    two_factor_auth:      './two_factor_auth.js',
     performance_bar:      './performance_bar.js',
     webpack_runtime:      './webpack.js',
   },
@@ -103,9 +111,12 @@ var config = {
         options: { limit: 2048 },
       },
       {
-        test: /\.(worker\.js|pdf|bmpr)$/,
+        test: /\.(worker(\.min)?\.js|pdf|bmpr)$/,
         exclude: /node_modules/,
         loader: 'file-loader',
+        options: {
+          name: '[name].[hash].[ext]',
+        }
       },
       {
         test: /locale\/\w+\/(.*)\.js$/,
@@ -185,7 +196,6 @@ var config = {
       name: 'common_d3',
       chunks: [
         'graphs',
-        'users',
         'monitoring',
       ],
     }),