Merge branch 'master' into expiration-date-on-memberships

14 files changed, 219 insertions, 100 deletions

diff --git a/config/application.rb b/config/application.rb
index 06ebb14a5fe..0c136623477 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -85,6 +85,8 @@ module Gitlab
     config.assets.precompile << "users/users_bundle.js"
     config.assets.precompile << "network/network_bundle.js"
     config.assets.precompile << "profile/profile_bundle.js"
+    config.assets.precompile << "boards/boards_bundle.js"
+    config.assets.precompile << "boards/test_utils/simulate_drag.js"
     config.assets.precompile << "lib/utils/*.js"
     config.assets.precompile << "lib/*.js"
     config.assets.precompile << "u2f.js"
@@ -107,7 +109,8 @@ module Gitlab
       end
     end
 
-    redis_config_hash = Gitlab::Redis.redis_store_options
+    # Use Redis caching across all environments
+    redis_config_hash = Gitlab::Redis.params
     redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
     redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
     config.cache_store = :redis_store, redis_config_hash
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 78e4b2b1bd3..54fa6a2c679 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -287,9 +287,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
 Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
-Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
-Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
+Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
+Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
 Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
diff --git a/config/initializers/5_backend.rb b/config/initializers/5_backend.rb
index e026151a032..ed88c8ee1b8 100644
--- a/config/initializers/5_backend.rb
+++ b/config/initializers/5_backend.rb
@@ -1,6 +1,3 @@
-# GIT over HTTP
-require_dependency Rails.root.join('lib/gitlab/backend/grack_auth')
-
 # GIT over SSH
 require_dependency Rails.root.join('lib/gitlab/backend/shell')
 
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 73977341b73..a0a8f88584c 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -100,6 +100,9 @@ Devise.setup do |config|
   # secure: true in order to force SSL only cookies.
   # config.cookie_options = {}
 
+  # Send a notification email when the user's password is changed
+  config.send_password_change_notification = true
+
   # ==> Configuration for :validatable
   # Range for password length. Default is 6..128.
   config.password_length = 8..128
diff --git a/config/initializers/metrics.rb b/config/initializers/metrics.rb
index f3cddac5b36..52522e099e7 100644
--- a/config/initializers/metrics.rb
+++ b/config/initializers/metrics.rb
@@ -144,6 +144,13 @@ if Gitlab::Metrics.enabled?
     end
 
     config.instrument_methods(Rinku)
+    config.instrument_instance_methods(Repository)
+
+    config.instrument_methods(Gitlab::Highlight)
+    config.instrument_instance_methods(Gitlab::Highlight)
+
+    # This is a Rails scope so we have to instrument it manually.
+    config.instrument_method(Project, :visible_to_user)
   end
 
   GC::Profiler.enable
diff --git a/config/initializers/mime_types.rb b/config/initializers/mime_types.rb
index 3e553120205..f498732feca 100644
--- a/config/initializers/mime_types.rb
+++ b/config/initializers/mime_types.rb
@@ -12,3 +12,10 @@ Mime::Type.register_alias "text/html",  :md
 Mime::Type.register "video/mp4",  :mp4, [], [:m4v, :mov]
 Mime::Type.register "video/webm", :webm
 Mime::Type.register "video/ogg",  :ogv
+
+middlewares = Gitlab::Application.config.middleware
+middlewares.swap(ActionDispatch::ParamsParser, ActionDispatch::ParamsParser, {
+  Mime::Type.lookup('application/vnd.git-lfs+json') => lambda do |body|
+    ActiveSupport::JSON.decode(body)
+  end
+})
diff --git a/config/initializers/request_profiler.rb b/config/initializers/request_profiler.rb
index fb5a7b8372e..a9aa802681a 100644
--- a/config/initializers/request_profiler.rb
+++ b/config/initializers/request_profiler.rb
@@ -1,3 +1,5 @@
+require 'gitlab/request_profiler/middleware'
+
 Rails.application.configure do |config|
   config.middleware.use(Gitlab::RequestProfiler::Middleware)
 end
diff --git a/config/initializers/secret_token.rb b/config/initializers/secret_token.rb
index dae3a4a9a93..291fa6c0abc 100644
--- a/config/initializers/secret_token.rb
+++ b/config/initializers/secret_token.rb
@@ -2,49 +2,86 @@
 
 require 'securerandom'
 
-# Your secret key for verifying the integrity of signed cookies.
-# If you change this key, all old signed cookies will become invalid!
-# Make sure the secret is at least 30 characters and all random,
-# no regular words or you'll be exposed to dictionary attacks.
-
-def find_secure_token
-  token_file = Rails.root.join('.secret')
-  if ENV.key?('SECRET_KEY_BASE')
-    ENV['SECRET_KEY_BASE']
-  elsif File.exist? token_file
-    # Use the existing token.
-    File.read(token_file).chomp
-  else
-    # Generate a new token of 64 random hexadecimal characters and store it in token_file.
-    token = SecureRandom.hex(64)
-    File.write(token_file, token)
-    token
+# Transition material in .secret to the secret_key_base key in config/secrets.yml.
+# Historically, ENV['SECRET_KEY_BASE'] takes precedence over .secret, so we maintain that
+# behavior.
+#
+# It also used to be the case that the key material in ENV['SECRET_KEY_BASE'] or .secret
+# was used to encrypt OTP (two-factor authentication) data so if present, we copy that key
+# material into config/secrets.yml under otp_key_base.
+#
+# Finally, if we have successfully migrated all secrets to config/secrets.yml, delete the
+# .secret file to avoid confusion.
+#
+def create_tokens
+  secret_file = Rails.root.join('.secret')
+  file_secret_key = File.read(secret_file).chomp if File.exist?(secret_file)
+  env_secret_key = ENV['SECRET_KEY_BASE']
+
+  # Ensure environment variable always overrides secrets.yml.
+  Rails.application.secrets.secret_key_base = env_secret_key if env_secret_key.present?
+
+  defaults = {
+    secret_key_base: file_secret_key || generate_new_secure_token,
+    otp_key_base: env_secret_key || file_secret_key || generate_new_secure_token,
+    db_key_base: generate_new_secure_token
+  }
+
+  missing_secrets = set_missing_keys(defaults)
+  write_secrets_yml(missing_secrets) unless missing_secrets.empty?
+
+  begin
+    File.delete(secret_file) if file_secret_key
+  rescue => e
+    warn "Error deleting useless .secret file: #{e}"
   end
 end
 
-Rails.application.config.secret_token = find_secure_token
-Rails.application.config.secret_key_base = find_secure_token
-
-# CI
 def generate_new_secure_token
   SecureRandom.hex(64)
 end
 
-if Rails.application.secrets.db_key_base.blank?
-  warn "Missing `db_key_base` for '#{Rails.env}' environment. The secrets will be generated and stored in `config/secrets.yml`"
+def warn_missing_secret(secret)
+  warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml."
+end
 
-  all_secrets = YAML.load_file('config/secrets.yml') if File.exist?('config/secrets.yml')
-  all_secrets ||= {}
+def set_missing_keys(defaults)
+  defaults.stringify_keys.each_with_object({}) do |(key, default), missing|
+    if Rails.application.secrets[key].blank?
+      warn_missing_secret(key)
 
-  # generate secrets
-  env_secrets = all_secrets[Rails.env.to_s] || {}
-  env_secrets['db_key_base'] ||= generate_new_secure_token
-  all_secrets[Rails.env.to_s] = env_secrets
+      missing[key] = Rails.application.secrets[key] = default
+    end
+  end
+end
+
+def write_secrets_yml(missing_secrets)
+  secrets_yml = Rails.root.join('config/secrets.yml')
+  rails_env = Rails.env.to_s
+  secrets = YAML.load_file(secrets_yml) if File.exist?(secrets_yml)
+  secrets ||= {}
+  secrets[rails_env] ||= {}
+
+  secrets[rails_env].merge!(missing_secrets) do |key, old, new|
+    # Previously, it was possible this was set to the literal contents of an Erb
+    # expression that evaluated to an empty value. We don't want to support that
+    # specifically, just ensure we don't break things further.
+    #
+    if old.present?
+      warn <<EOM
+Rails.application.secrets.#{key} was blank, but the literal value in config/secrets.yml was:
+  #{old}
 
-  # save secrets
-  File.open('config/secrets.yml', 'w', 0600) do |file|
-    file.write(YAML.dump(all_secrets))
+This probably isn't the expected value for this secret. To keep using a literal Erb string in config/secrets.yml, replace `<%` with `<%%`.
+EOM
+
+      exit 1 # rubocop:disable Rails/Exit
+    end
+
+    new
   end
 
-  Rails.application.secrets.db_key_base = env_secrets['db_key_base']
+  File.write(secrets_yml, YAML.dump(secrets), mode: 'w', perm: 0600)
 end
+
+create_tokens
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index 0d9d87bac00..70be2617cab 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -13,9 +13,9 @@ end
 if Rails.env.test?
   Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
 else
-  redis_config = Gitlab::Redis.redis_store_options
+  redis_config = Gitlab::Redis.params
   redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
-  
+
   Gitlab::Application.config.session_store(
     :redis_store, # Using the cookie_store would enable session replay attacks.
     servers: redis_config,
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index cf49ec2194c..f7e714cd6bc 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,8 +1,9 @@
+# Custom Redis configuration
+redis_config_hash = Gitlab::Redis.params
+redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
+
 Sidekiq.configure_server do |config|
-  config.redis = {
-    url: Gitlab::Redis.url,
-    namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
-  }
+  config.redis = redis_config_hash
 
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
@@ -39,8 +40,5 @@ Sidekiq.configure_server do |config|
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = {
-    url: Gitlab::Redis.url,
-    namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
-  }
+  config.redis = redis_config_hash
 end
diff --git a/config/initializers/trusted_proxies.rb b/config/initializers/trusted_proxies.rb
index 30770b71e24..cd869657c53 100644
--- a/config/initializers/trusted_proxies.rb
+++ b/config/initializers/trusted_proxies.rb
@@ -7,6 +7,8 @@ module Rack
   class Request
     def trusted_proxy?(ip)
       Rails.application.config.action_dispatch.trusted_proxies.any? { |proxy| proxy === ip }
+    rescue IPAddr::InvalidAddressError
+      false
     end
   end
 end
diff --git a/config/mail_room.yml b/config/mail_room.yml
index 7cab24b295e..c639f8260aa 100644
--- a/config/mail_room.yml
+++ b/config/mail_room.yml
@@ -1,47 +1,36 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
 :mailboxes:
-<%
-require "yaml"
-require "json"
-require_relative "lib/gitlab/redis" unless defined?(Gitlab::Redis)
+  <%
+    require_relative "lib/gitlab/mail_room" unless defined?(Gitlab::MailRoom)
+    config = Gitlab::MailRoom.config
 
-rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
-
-config_file = ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] || "config/gitlab.yml"
-if File.exists?(config_file)
-  all_config = YAML.load_file(config_file)[rails_env]
-
-  config = all_config["incoming_email"] || {}
-  config['enabled']    = false    if config['enabled'].nil?
-  config['port']       = 143      if config['port'].nil?
-  config['ssl']        = false    if config['ssl'].nil?
-  config['start_tls']  = false    if config['start_tls'].nil?
-  config['mailbox']    = "inbox"  if config['mailbox'].nil?
-
-  if config['enabled'] && config['address']
-    redis_url = Gitlab::Redis.new(rails_env).url
-    %>
+    if Gitlab::MailRoom.enabled?
+  %>
     -
-      :host: <%= config['host'].to_json %>
-      :port: <%= config['port'].to_json %>
-      :ssl: <%= config['ssl'].to_json %>
-      :start_tls: <%= config['start_tls'].to_json %>
-      :email: <%= config['user'].to_json %>
-      :password: <%= config['password'].to_json %>
+      :host: <%= config[:host].to_json %>
+      :port: <%= config[:port].to_json %>
+      :ssl: <%= config[:ssl].to_json %>
+      :start_tls: <%= config[:start_tls].to_json %>
+      :email: <%= config[:user].to_json %>
+      :password: <%= config[:password].to_json %>
+      :idle_timeout: 60
 
-      :name: <%= config['mailbox'].to_json %>
+      :name: <%= config[:mailbox].to_json %>
 
       :delete_after_delivery: true
 
       :delivery_method: sidekiq
       :delivery_options:
-        :redis_url: <%= redis_url.to_json %>
-        :namespace: resque:gitlab
+        :redis_url: <%= config[:redis_url].to_json %>
+        :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
         :queue: incoming_email
         :worker: EmailReceiverWorker
 
       :arbitration_method: redis
       :arbitration_options:
-        :redis_url: <%= redis_url.to_json %>
-        :namespace: mail_room:gitlab
+        :redis_url: <%= config[:redis_url].to_json %>
+        :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
+
   <% end %>
-<% end %>
diff --git a/config/resque.yml.example b/config/resque.yml.example
index d98f43f71b2..0c19d8bc1d3 100644
--- a/config/resque.yml.example
+++ b/config/resque.yml.example
@@ -1,6 +1,34 @@
 # If you change this file in a Merge Request, please also create
 # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
 #
-development: redis://localhost:6379
-test: redis://localhost:6379
-production: unix:/var/run/redis/redis.sock
+development:
+  url: redis://localhost:6379
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6379
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26379 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26379 # point to sentinel, not to redis port
diff --git a/config/routes.rb b/config/routes.rb
index 308d83af57e..9c74df31de5 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -84,9 +84,6 @@ Rails.application.routes.draw do
   # Health check
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
 
-  # Enable Grack support (for LFS only)
-  mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\/(info\/lfs|gitlab-lfs)/.match(request.path_info) }, via: [:get, :post, :put]
-
   # Help
   get 'help'           => 'help#index'
   get 'help/shortcuts' => 'help#shortcuts'
@@ -255,7 +252,11 @@ Rails.application.routes.draw do
     resource :impersonation, only: :destroy
 
     resources :abuse_reports, only: [:index, :destroy]
-    resources :spam_logs, only: [:index, :destroy]
+    resources :spam_logs, only: [:index, :destroy] do
+      member do
+        post :mark_as_ham
+      end
+    end
 
     resources :applications
 
@@ -471,7 +472,7 @@ Rails.application.routes.draw do
         post :unarchive
         post :housekeeping
         post :toggle_star
-        post :markdown_preview
+        post :preview_markdown
         post :export
         post :remove_export
         post :generate_new_export
@@ -482,11 +483,26 @@ Rails.application.routes.draw do
       end
 
       scope module: :projects do
-        # Git HTTP clients ('git clone' etc.)
         scope constraints: { id: /.+\.git/, format: nil } do
+          # Git HTTP clients ('git clone' etc.)
           get '/info/refs', to: 'git_http#info_refs'
           post '/git-upload-pack', to: 'git_http#git_upload_pack'
           post '/git-receive-pack', to: 'git_http#git_receive_pack'
+
+          # Git LFS API (metadata)
+          post '/info/lfs/objects/batch', to: 'lfs_api#batch'
+          post '/info/lfs/objects', to: 'lfs_api#deprecated'
+          get '/info/lfs/objects/*oid', to: 'lfs_api#deprecated'
+
+          # GitLab LFS object storage
+          scope constraints: { oid: /[a-f0-9]{64}/ } do
+            get '/gitlab-lfs/objects/*oid', to: 'lfs_storage#download'
+
+            scope constraints: { size: /[0-9]+/ } do
+              put '/gitlab-lfs/objects/*oid/*size/authorize', to: 'lfs_storage#upload_authorize'
+              put '/gitlab-lfs/objects/*oid/*size', to: 'lfs_storage#upload_finalize'
+            end
+          end
         end
 
         # Allow /info/refs, /info/refs?service=git-upload-pack, and
@@ -512,6 +528,11 @@ Rails.application.routes.draw do
         put '/update/*id', to: 'blob#update', constraints: { id: /.+/ }, as: 'update_blob'
         post '/preview/*id', to: 'blob#preview', constraints: { id: /.+/ }, as: 'preview_blob'
 
+        #
+        # Templates
+        #
+        get '/templates/:template_type/:key' => 'templates#show', as: :template
+
         scope do
           get(
             '/blob/*id/diff',
@@ -626,13 +647,17 @@ Rails.application.routes.draw do
 
         get '/compare/:from...:to', to: 'compare#show', as: 'compare', constraints: { from: /.+/, to: /.+/ }
 
-        resources :network, only: [:show], constraints: { id: /(?:[^.]|\.(?!json$))+/, format: /json/ }
+        # Don't use format parameter as file extension (old 3.0.x behavior)
+        # See http://guides.rubyonrails.org/routing.html#route-globbing-and-wildcard-segments
+        scope format: false do
+          resources :network, only: [:show], constraints: { id: Gitlab::Regex.git_reference_regex }
 
-        resources :graphs, only: [:show], constraints: { id: /(?:[^.]|\.(?!json$))+/, format: /json/ } do
-          member do
-            get :commits
-            get :ci
-            get :languages
+          resources :graphs, only: [:show], constraints: { id: Gitlab::Regex.git_reference_regex } do
+            member do
+              get :commits
+              get :ci
+              get :languages
+            end
           end
         end
 
@@ -656,7 +681,7 @@ Rails.application.routes.draw do
           get '/wikis/*id', to: 'wikis#show', as: 'wiki', constraints: WIKI_SLUG_ID
           delete '/wikis/*id', to: 'wikis#destroy', constraints: WIKI_SLUG_ID
           put '/wikis/*id', to: 'wikis#update', constraints: WIKI_SLUG_ID
-          post '/wikis/*id/markdown_preview', to: 'wikis#markdown_preview', constraints: WIKI_SLUG_ID, as: 'wiki_markdown_preview'
+          post '/wikis/*id/preview_markdown', to: 'wikis#preview_markdown', constraints: WIKI_SLUG_ID, as: 'wiki_preview_markdown'
         end
 
         resource :repository, only: [:create] do
@@ -702,7 +727,9 @@ Rails.application.routes.draw do
           member do
             get :commits
             get :diffs
+            get :conflicts
             get :builds
+            get :pipelines
             get :merge_check
             post :merge
             post :cancel_merge_when_build_succeeds
@@ -711,6 +738,7 @@ Rails.application.routes.draw do
             post :toggle_award_emoji
             post :remove_wip
             get :diff_for_path
+            post :resolve_conflicts
           end
 
           collection do
@@ -741,7 +769,7 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :environments, only: [:index, :show, :new, :create, :destroy]
+        resources :environments
 
         resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
           collection do
@@ -797,6 +825,7 @@ Rails.application.routes.draw do
           member do
             post :toggle_subscription
             post :toggle_award_emoji
+            post :mark_as_spam
             get :referenced_merge_requests
             get :related_branches
             get :can_create_branch
@@ -830,6 +859,20 @@ Rails.application.routes.draw do
           end
         end
 
+        resource :board, only: [:show] do
+          scope module: :boards do
+            resources :issues, only: [:update]
+
+            resources :lists, only: [:index, :create, :update, :destroy] do
+              collection do
+                post :generate
+              end
+
+              resources :issues, only: [:index]
+            end
+          end
+        end
+
         resources :todos, only: [:create]
 
         resources :uploads, only: [:create] do
@@ -853,7 +896,10 @@ Rails.application.routes.draw do
         resources :badges, only: [:index] do
           collection do
             scope '*ref', constraints: { ref: Gitlab::Regex.git_reference_regex } do
-              get :build, constraints: { format: /svg/ }
+              constraints format: /svg/ do
+                get :build
+                get :coverage
+              end
             end
           end
         end