Merge branch 'master' into artifacts-from-ref-and-build-name-api

* master: (261 commits)
  Add link to user profile to commit avatar (!5163)
  Refactor service settings view
  Fix a problem with processing a pipeline where stage only has manual actions
  A CHANGELOG entry
  Don't show other actions of the same name
  Use limit parameter rather than hardcoded value
  Remove icons from explore nav
  Change how we style redirect_to
  Use flash[:notice] only
  Create PipelinesSettingsController for showing settings page
  Fix a few nitpicks
  Allow to disable user request access to groups/projects
  Enable Style/MultilineTernaryOperator rubocop cop
  Fix review comments
  Update routes
  Update CHANGELOG
  Improve implementation of variables
  Log cron_jobs configuration instead of raising exception
  Added checks for migration downtime
  Ensure to_json methods take optional argument
  ...

21 files changed, 266 insertions, 58 deletions

diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index acb4812b5cf..4df6ca8333e 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -24,7 +24,7 @@ module API
 
         pipelines = user_project.pipelines.where(sha: params[:sha])
         statuses = ::CommitStatus.where(pipeline: pipelines)
-        statuses = statuses.latest unless parse_boolean(params[:all])
+        statuses = statuses.latest unless to_boolean(params[:all])
         statuses = statuses.where(ref: params[:ref]) if params[:ref].present?
         statuses = statuses.where(stage: params[:stage]) if params[:stage].present?
         statuses = statuses.where(name: params[:name]) if params[:name].present?
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index d6e4eb2afd7..130509cdad6 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -5,10 +5,6 @@ module API
     SUDO_HEADER = "HTTP_SUDO"
     SUDO_PARAM = :sudo
 
-    def parse_boolean(value)
-      [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value)
-    end
-
     def to_boolean(value)
       return true if value =~ /^(true|t|yes|y|1|on)$/i
       return false if value =~ /^(false|f|no|n|0|off)$/i
@@ -297,7 +293,7 @@ module API
     def filter_projects(projects)
       # If the archived parameter is passed, limit results accordingly
       if params[:archived].present?
-        projects = projects.where(archived: parse_boolean(params[:archived]))
+        projects = projects.where(archived: to_boolean(params[:archived]))
       end
 
       if params[:search].present?
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 4fcdf8968c9..2b685621da9 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -242,7 +242,7 @@ module API
             should_remove_source_branch: params[:should_remove_source_branch]
           }
 
-          if parse_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active?
+          if to_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active?
             ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params).
               execute(merge_request)
           else
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 6d2a6f3946c..8fed7db8803 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -8,7 +8,7 @@ module API
         def map_public_to_visibility_level(attrs)
           publik = attrs.delete(:public)
           if publik.present? && !attrs[:visibility_level].present?
-            publik = parse_boolean(publik)
+            publik = to_boolean(publik)
             # Since setting the public attribute to private could mean either
             # private or internal, use the more conservative option, private.
             attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE
diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb
index c78da404607..21ed0410f7f 100644
--- a/lib/banzai/filter/relative_link_filter.rb
+++ b/lib/banzai/filter/relative_link_filter.rb
@@ -112,8 +112,7 @@ module Banzai
       end
 
       def current_commit
-        @current_commit ||= context[:commit] ||
-          ref ? repository.commit(ref) : repository.head_commit
+        @current_commit ||= context[:commit] || ref ? repository.commit(ref) : repository.head_commit
       end
 
       def relative_url_root
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 41449d720b3..83afed9f49f 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -44,23 +44,51 @@ module Ci
     end
 
     def builds_for_ref(ref, tag = false, trigger_request = nil)
-      jobs_for_ref(ref, tag, trigger_request).map do |name, job|
-        build_job(name, job)
+      jobs_for_ref(ref, tag, trigger_request).map do |name, _|
+        build_attributes(name)
       end
     end
 
     def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, job|
-        build_job(name, job)
+      jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _|
+        build_attributes(name)
       end
     end
 
     def builds
-      @jobs.map do |name, job|
-        build_job(name, job)
+      @jobs.map do |name, _|
+        build_attributes(name)
       end
     end
 
+    def build_attributes(name)
+      job = @jobs[name.to_sym] || {}
+      {
+        stage_idx: @stages.index(job[:stage]),
+        stage: job[:stage],
+        ##
+        # Refactoring note:
+        #  - before script behaves differently than after script
+        #  - after script returns an array of commands
+        #  - before script should be a concatenated command
+        commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
+        tag_list: job[:tags] || [],
+        name: name,
+        allow_failure: job[:allow_failure] || false,
+        when: job[:when] || 'on_success',
+        environment: job[:environment],
+        yaml_variables: yaml_variables(name),
+        options: {
+          image: job[:image] || @image,
+          services: job[:services] || @services,
+          artifacts: job[:artifacts],
+          cache: job[:cache] || @cache,
+          dependencies: job[:dependencies],
+          after_script: job[:after_script] || @after_script,
+        }.compact
+      }
+    end
+
     private
 
     def initial_parsing
@@ -89,33 +117,6 @@ module Ci
       @jobs[name] = { stage: stage }.merge(job)
     end
 
-    def build_job(name, job)
-      {
-        stage_idx: @stages.index(job[:stage]),
-        stage: job[:stage],
-        ##
-        # Refactoring note:
-        #  - before script behaves differently than after script
-        #  - after script returns an array of commands
-        #  - before script should be a concatenated command
-        commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"),
-        tag_list: job[:tags] || [],
-        name: name,
-        allow_failure: job[:allow_failure] || false,
-        when: job[:when] || 'on_success',
-        environment: job[:environment],
-        yaml_variables: yaml_variables(name),
-        options: {
-          image: job[:image] || @image,
-          services: job[:services] || @services,
-          artifacts: job[:artifacts],
-          cache: job[:cache] || @cache,
-          dependencies: job[:dependencies],
-          after_script: job[:after_script] || @after_script,
-        }.compact
-      }
-    end
-
     def yaml_variables(name)
       variables = global_variables.merge(job_variables(name))
       variables.map do |key, value|
diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb
index dfa83a0eab3..5fe86553bd0 100644
--- a/lib/gitlab/checks/force_push.rb
+++ b/lib/gitlab/checks/force_push.rb
@@ -8,8 +8,8 @@ module Gitlab
         if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev)
           false
         else
-          missed_refs, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list #{oldrev} ^#{newrev}))
-          missed_refs.split("\n").size > 0
+          missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev}))
+          missed_ref.present?
         end
       end
     end
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 989fff8918e..2fdcf8d7838 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -73,8 +73,8 @@ module Gitlab
           diff_refs.complete?
       end
 
-      def to_json
-        JSON.generate(self.to_h)
+      def to_json(opts = nil)
+        JSON.generate(self.to_h, opts)
       end
 
       def type
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
new file mode 100644
index 00000000000..ab9537ed7d7
--- /dev/null
+++ b/lib/gitlab/downtime_check.rb
@@ -0,0 +1,71 @@
+module Gitlab
+  # Checks if a set of migrations requires downtime or not.
+  class DowntimeCheck
+    # The constant containing the boolean that indicates if downtime is needed
+    # or not.
+    DOWNTIME_CONST = :DOWNTIME
+
+    # The constant that specifies the reason for the migration requiring
+    # downtime.
+    DOWNTIME_REASON_CONST = :DOWNTIME_REASON
+
+    # Checks the given migration paths and returns an Array of
+    # `Gitlab::DowntimeCheck::Message` instances.
+    #
+    # migrations - The migration file paths to check.
+    def check(migrations)
+      migrations.map do |path|
+        require(path)
+
+        migration_class = class_for_migration_file(path)
+
+        unless migration_class.const_defined?(DOWNTIME_CONST)
+          raise "The migration in #{path} does not specify if it requires " \
+            "downtime or not"
+        end
+
+        if online?(migration_class)
+          Message.new(path)
+        else
+          reason = downtime_reason(migration_class)
+
+          unless reason
+            raise "The migration in #{path} requires downtime but no reason " \
+              "was given"
+          end
+
+          Message.new(path, true, reason)
+        end
+      end
+    end
+
+    # Checks the given migrations and prints the results to STDOUT/STDERR.
+    #
+    # migrations - The migration file paths to check.
+    def check_and_print(migrations)
+      check(migrations).each do |message|
+        puts message.to_s # rubocop: disable Rails/Output
+      end
+    end
+
+    # Returns the class for the given migration file path.
+    def class_for_migration_file(path)
+      File.basename(path, File.extname(path)).split('_', 2).last.camelize.
+        constantize
+    end
+
+    # Returns true if the given migration can be performed without downtime.
+    def online?(migration)
+      migration.const_get(DOWNTIME_CONST) == false
+    end
+
+    # Returns the downtime reason, or nil if none was defined.
+    def downtime_reason(migration)
+      if migration.const_defined?(DOWNTIME_REASON_CONST)
+        migration.const_get(DOWNTIME_REASON_CONST)
+      else
+        nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
new file mode 100644
index 00000000000..4446e921e0d
--- /dev/null
+++ b/lib/gitlab/downtime_check/message.rb
@@ -0,0 +1,28 @@
+module Gitlab
+  class DowntimeCheck
+    class Message
+      attr_reader :path, :offline, :reason
+
+      OFFLINE = "\e[32moffline\e[0m"
+      ONLINE = "\e[31monline\e[0m"
+
+      # path - The file path of the migration.
+      # offline - When set to `true` the migration will require downtime.
+      # reason - The reason as to why the migration requires downtime.
+      def initialize(path, offline = false, reason = nil)
+        @path = path
+        @offline = offline
+        @reason = reason
+      end
+
+      def to_s
+        label = offline ? OFFLINE : ONLINE
+
+        message = "[#{label}]: #{path}"
+        message += ": #{reason}" if reason
+
+        message
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 308f23bc9bc..8e8f39d9cb2 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -110,6 +110,7 @@ module Gitlab
 
     def deploy_key_can_read_project?
       if deploy_key
+        return true if project.public?
         deploy_key.projects.include?(project)
       else
         false
diff --git a/lib/gitlab/git_access_status.rb b/lib/gitlab/git_access_status.rb
index 5a806ff6e0d..09bb01be694 100644
--- a/lib/gitlab/git_access_status.rb
+++ b/lib/gitlab/git_access_status.rb
@@ -8,8 +8,8 @@ module Gitlab
       @message = message
     end
 
-    def to_json
-      { status: @status, message: @message }.to_json
+    def to_json(opts = nil)
+      { status: @status, message: @message }.to_json(opts)
     end
   end
 end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index d4f12cb1df9..c5a11148d33 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -5,7 +5,7 @@ module Gitlab
       gon.default_avatar_url     = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s
       gon.max_file_size          = current_application_settings.max_attachment_size
       gon.relative_url_root      = Gitlab.config.gitlab.relative_url_root
-      gon.shortcuts_path         = help_shortcuts_path
+      gon.shortcuts_path         = help_page_path('shortcuts')
       gon.user_color_scheme      = Gitlab::ColorSchemes.for_user(current_user).css_class
       gon.award_menu_url         = emojis_path
 
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
new file mode 100644
index 00000000000..352539eb594
--- /dev/null
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module ImportExport
+    class AvatarRestorer
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def restore
+        return true unless avatar_export_file
+
+        @project.avatar = File.open(avatar_export_file)
+        @project.save!
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def avatar_export_file
+        @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
+      end
+
+      def avatar_export_path
+        File.join(@shared.export_path, 'avatar')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb
new file mode 100644
index 00000000000..998c21e2586
--- /dev/null
+++ b/lib/gitlab/import_export/avatar_saver.rb
@@ -0,0 +1,31 @@
+module Gitlab
+  module ImportExport
+    class AvatarSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return true unless @project.avatar.exists?
+
+        copy_files(avatar_path, avatar_export_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def avatar_export_path
+        File.join(@shared.export_path, 'avatar', @project.avatar_identifier)
+      end
+
+      def avatar_path
+        @project.avatar.path
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 2249904145c..5dd0e34c18e 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -36,6 +36,15 @@ module Gitlab
       def git_bin_path
         Gitlab.config.git.bin_path
       end
+
+      def copy_files(source, destination)
+        # if we are copying files, create the destination folder
+        destination_folder = File.file?(source) ? File.dirname(destination) : destination
+
+        FileUtils.mkdir_p(destination_folder)
+        FileUtils.copy_entry(source, destination)
+        true
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 6b69a653f12..e9ee47fc090 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def execute
-        if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && [project_tree, avatar_restorer, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
@@ -35,6 +35,10 @@ module Gitlab
                                                                         project: @project)
       end
 
+      def avatar_restorer
+        Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
       def repo_restorer
         Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
                                                shared: @shared,
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index d6f4fa57510..62a2553675c 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module ImportExport
     class UploadsSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
       def initialize(project:, shared:)
         @project = project
         @shared = shared
@@ -17,12 +19,6 @@ module Gitlab
 
       private
 
-      def copy_files(source, destination)
-        FileUtils.mkdir_p(destination)
-        FileUtils.copy_entry(source, destination)
-        true
-      end
-
       def uploads_export_path
         File.join(@shared.export_path, 'uploads')
       end
diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake
new file mode 100644
index 00000000000..30a2e9be5ce
--- /dev/null
+++ b/lib/tasks/downtime_check.rake
@@ -0,0 +1,26 @@
+desc 'Checks if migrations in a branch require downtime'
+task downtime_check: :environment do
+  # First we'll want to make sure we're comparing with the right upstream
+  # repository/branch.
+  current_branch = `git rev-parse --abbrev-ref HEAD`.strip
+
+  # Either the developer ran this task directly on the master branch, or they're
+  # making changes directly on the master branch.
+  if current_branch == 'master'
+    if defined?(Gitlab::License)
+      repo = 'gitlab-ee'
+    else
+      repo = 'gitlab-ce'
+    end
+
+    `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
+
+    compare_with = 'FETCH_HEAD'
+  # The developer is working on a different branch, in this case we can just
+  # compare with the master branch.
+  else
+    compare_with = 'master'
+  end
+
+  Rake::Task['gitlab:db:downtime_check'].invoke(compare_with)
+end
diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake
index e9a4e37ec48..60f4636e737 100644
--- a/lib/tasks/gitlab/check.rake
+++ b/lib/tasks/gitlab/check.rake
@@ -784,7 +784,7 @@ namespace :gitlab do
       servers.each do |server|
         puts "Server: #{server}"
         Gitlab::LDAP::Adapter.open(server) do |adapter|
-          users = adapter.users(adapter.config.uid, '*', 100)
+          users = adapter.users(adapter.config.uid, '*', limit)
           users.each do |user|
             puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}"
           end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 7230b9485be..0ec19e1a625 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -46,5 +46,20 @@ namespace :gitlab do
         Rake::Task['db:seed_fu'].invoke
       end
     end
+
+    desc 'Checks if migrations require downtime or not'
+    task :downtime_check, [:ref] => :environment do |_, args|
+      abort 'You must specify a Git reference to compare with' unless args[:ref]
+
+      require 'shellwords'
+
+      ref = Shellwords.escape(args[:ref])
+
+      migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines.
+        map { |file| Rails.root.join(file.strip).to_s }.
+        select { |file| File.file?(file) }
+
+      Gitlab::DowntimeCheck.new.check_and_print(migrations)
+    end
   end
 end