Merge branch 'master' into refactor/ci-config-move-global-entries

* master: (352 commits)
  Display last commit of deleted branch in push events (!4699)
  add changelog
  add missing attribute to attr_encrypted so it is fully backwards-compatible
  Add "GitLab team members only" to diagram link
  doc: note that .gitattributes uses default branch
  use the conf lexer so we have highlighted comments
  first draft of docs
  support cgi style options, such as erb?parent=json
  move the path alias to a more appropriate location
  make #custom_language private
  appease rubocop
  add an alias for Snippet#path
  appease rubocop
  check the tag so that an instance will pass too
  fix the spec, using project.change_head
  Revert "bump the master sha for gitlab-test!9"
  bump the master sha for gitlab-test!9
  add custom highlighting via .gitattributes
  Rename Licenses API to License Templates API
  Check for conflict with wiki projects when creating a new project.
  ...

17 files changed, 178 insertions, 77 deletions

diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb
index 6d0e30e916f..831f1e635ba 100644
--- a/lib/gitlab/access.rb
+++ b/lib/gitlab/access.rb
@@ -5,6 +5,8 @@
 #
 module Gitlab
   module Access
+    class AccessDeniedError < StandardError; end
+
     GUEST     = 10
     REPORTER  = 20
     DEVELOPER = 30
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 7e3f5abba62..ab7b811c5d8 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -31,7 +31,7 @@ module Grack
 
       auth!
 
-      lfs_response = Gitlab::Lfs::Router.new(project, @user, @request).try_call
+      lfs_response = Gitlab::Lfs::Router.new(project, @user, @ci, @request).try_call
       return lfs_response unless lfs_response.nil?
 
       if @user.nil? && !@ci
diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb
index 997a22779a0..d62bc50ce78 100644
--- a/lib/gitlab/blame.rb
+++ b/lib/gitlab/blame.rb
@@ -41,7 +41,8 @@ module Gitlab
 
     def highlighted_lines
       @blob.load_all_data!(repository)
-      @highlighted_lines ||= Gitlab::Highlight.highlight(@blob.name, @blob.data).lines
+      @highlighted_lines ||=
+        Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines
     end
 
     def project
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index e2fee6b9f3e..047c77c6fc2 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -37,7 +37,7 @@ module Gitlab
         end
 
         def diffs
-          @diffs ||= (safe_diff_files(compare.diffs, diff_refs) if compare)
+          @diffs ||= (safe_diff_files(compare.diffs(max_files: 30), diff_refs) if compare)
         end
 
         def diffs_count
diff --git a/lib/gitlab/gitignore.rb b/lib/gitlab/gitignore.rb
deleted file mode 100644
index f46b43b61a4..00000000000
--- a/lib/gitlab/gitignore.rb
+++ /dev/null
@@ -1,56 +0,0 @@
-module Gitlab
-  class Gitignore
-    FILTER_REGEX = /\.gitignore\z/.freeze
-
-    def initialize(path)
-      @path = path
-    end
-
-    def name
-      File.basename(@path, '.gitignore')
-    end
-
-    def content
-      File.read(@path)
-    end
-
-    class << self
-      def all
-        languages_frameworks + global
-      end
-
-      def find(key)
-        file_name = "#{key}.gitignore"
-
-        directory = select_directory(file_name)
-        directory ? new(File.join(directory, file_name)) : nil
-      end
-
-      def global
-        files_for_folder(global_dir).map { |file| new(File.join(global_dir, file)) }
-      end
-
-      def languages_frameworks
-        files_for_folder(gitignore_dir).map { |file| new(File.join(gitignore_dir, file)) }
-      end
-
-      private
-
-      def select_directory(file_name)
-        [gitignore_dir, global_dir].find { |dir| File.exist?(File.join(dir, file_name)) }
-      end
-
-      def global_dir
-        File.join(gitignore_dir, 'Global')
-      end
-
-      def gitignore_dir
-        Rails.root.join('vendor/gitignore')
-      end
-
-      def files_for_folder(dir)
-        Dir.glob("#{dir.to_s}/*.gitignore").map { |file| file.gsub(FILTER_REGEX, '') }
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 280120b0f9e..41296415e35 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -1,7 +1,7 @@
 module Gitlab
   class Highlight
-    def self.highlight(blob_name, blob_content, nowrap: true, plain: false)
-      new(blob_name, blob_content, nowrap: nowrap).
+    def self.highlight(blob_name, blob_content, repository: nil, nowrap: true, plain: false)
+      new(blob_name, blob_content, nowrap: nowrap, repository: repository).
         highlight(blob_content, continue: false, plain: plain)
     end
 
@@ -10,12 +10,21 @@ module Gitlab
       return [] unless blob
 
       blob.load_all_data!(repository)
-      highlight(file_name, blob.data).lines.map!(&:html_safe)
+      highlight(file_name, blob.data, repository: repository).lines.map!(&:html_safe)
     end
 
-    def initialize(blob_name, blob_content, nowrap: true)
+    attr_reader :lexer
+    def initialize(blob_name, blob_content, repository: nil, nowrap: true)
+      @blob_name = blob_name
+      @blob_content = blob_content
+      @repository = repository
       @formatter = rouge_formatter(nowrap: nowrap)
-      @lexer = Rouge::Lexer.guess(filename: blob_name, source: blob_content).new rescue Rouge::Lexers::PlainText
+
+      @lexer = custom_language || begin
+        Rouge::Lexer.guess(filename: blob_name, source: blob_content).new
+      rescue Rouge::Lexer::AmbiguousGuess => e
+        e.alternatives.sort_by(&:tag).first
+      end
     end
 
     def highlight(text, continue: true, plain: false)
@@ -30,6 +39,14 @@ module Gitlab
 
     private
 
+    def custom_language
+      language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language')
+
+      return nil unless language_name
+
+      Rouge::Lexer.find_fancy(language_name)
+    end
+
     def rouge_formatter(options = {})
       options = options.reverse_merge(
         nowrap: true,
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 0e70d9282d5..82d1e1805c5 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -23,7 +23,11 @@ module Gitlab
       private
 
       def decompress_archive
-        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+        result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
+
+        raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
+
+        true
       end
     end
   end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index d209e04f7be..595b20a09bd 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -10,17 +10,22 @@ module Gitlab
       end
 
       def execute
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
-                                                  shared: @shared)
-        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
         end
+
+        remove_import_file
       end
 
       private
 
+      def import_file
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+      end
+
       def check_version!
         Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
       end
@@ -59,6 +64,10 @@ module Gitlab
       def wiki_repo_path
         File.join(@shared.export_path, 'project.wiki.bundle')
       end
+
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index b872780f20a..92bf7e0a2fc 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -12,6 +12,8 @@ module Gitlab
 
       USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
 
+      BUILD_MODELS = %w[Ci::Build commit_status].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -70,7 +72,7 @@ module Gitlab
       end
 
       def generate_imported_object
-        if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
+        if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes
           trace = @relation_hash.delete('trace')
           imported_object do |object|
             object.trace = trace
diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb
index 9d9617761b3..e3ed2f6791d 100644
--- a/lib/gitlab/lfs/response.rb
+++ b/lib/gitlab/lfs/response.rb
@@ -2,10 +2,11 @@ module Gitlab
   module Lfs
     class Response
 
-      def initialize(project, user, request)
+      def initialize(project, user, ci, request)
         @origin_project = project
         @project = storage_project(project)
         @user = user
+        @ci = ci
         @env = request.env
         @request = request
       end
@@ -189,7 +190,7 @@ module Gitlab
         return render_not_enabled unless Gitlab.config.lfs.enabled
 
         unless @project.public?
-          return render_unauthorized unless @user
+          return render_unauthorized unless @user || @ci
           return render_forbidden unless user_can_fetch?
         end
 
@@ -210,7 +211,7 @@ module Gitlab
 
       def user_can_fetch?
         # Check user access against the project they used to initiate the pull
-        @user.can?(:download_code, @origin_project)
+        @ci || @user.can?(:download_code, @origin_project)
       end
 
       def user_can_push?
diff --git a/lib/gitlab/lfs/router.rb b/lib/gitlab/lfs/router.rb
index 78d02891102..69bd5e62305 100644
--- a/lib/gitlab/lfs/router.rb
+++ b/lib/gitlab/lfs/router.rb
@@ -1,9 +1,12 @@
 module Gitlab
   module Lfs
     class Router
-      def initialize(project, user, request)
+      attr_reader :project, :user, :ci, :request
+
+      def initialize(project, user, ci, request)
         @project = project
         @user = user
+        @ci = ci
         @env = request.env
         @request = request
       end
@@ -80,7 +83,7 @@ module Gitlab
       def lfs
         return unless @project
 
-        Gitlab::Lfs::Response.new(@project, @user, @request)
+        Gitlab::Lfs::Response.new(@project, @user, @ci, @request)
       end
 
       def sanitize_tmp_filename(name)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index fd98aa3412e..a1240fd33ee 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -8,6 +8,8 @@ module Gitlab
         trans = Transaction.new("#{worker.class.name}#perform")
 
         begin
+          # Old gitlad-shell messages don't provide enqueued_at/created_at attributes
+          trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
           trans.run { yield }
         ensure
           trans.finish
diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb
index 78f3ecb4cb4..7af75a9cc4c 100644
--- a/lib/gitlab/o_auth/user.rb
+++ b/lib/gitlab/o_auth/user.rb
@@ -74,7 +74,7 @@ module Gitlab
         if user
           # Case when a LDAP user already exists in Gitlab. Add the OAuth identity to existing account.
           log.info "LDAP account found for user #{user.username}. Building new #{auth_hash.provider} identity."
-          user.identities.build(extern_uid: auth_hash.uid, provider: auth_hash.provider)
+          user.identities.find_or_initialize_by(extern_uid: auth_hash.uid, provider: auth_hash.provider)
         else
           log.info "No existing LDAP account was found in GitLab. Checking for #{auth_hash.provider} account."
           user = find_by_uid_and_provider
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index ae85b294d31..4831c46c4be 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -25,7 +25,7 @@ module Gitlab
 
           Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\
             "#{MAX_RSS}"
-          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} "\
+          Sidekiq.logger.warn "this thread will shut down PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"\
             "in #{GRACE_TIME} seconds"
           sleep(GRACE_TIME)
 
@@ -36,7 +36,7 @@ module Gitlab
             "#{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
           sleep(SHUTDOWN_WAIT)
 
-          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid}"
+          Sidekiq.logger.warn "sending #{SHUTDOWN_SIGNAL} to PID #{Process.pid} - Worker #{worker.class} - JID-#{job['jid']}"
           Process.kill(SHUTDOWN_SIGNAL, Process.pid)
         end
       end
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
new file mode 100644
index 00000000000..760ff3e614a
--- /dev/null
+++ b/lib/gitlab/template/base_template.rb
@@ -0,0 +1,67 @@
+module Gitlab
+  module Template
+    class BaseTemplate
+      def initialize(path)
+        @path = path
+      end
+
+      def name
+        File.basename(@path, self.class.extension)
+      end
+
+      def content
+        File.read(@path)
+      end
+
+      class << self
+        def all
+          self.categories.keys.flat_map { |cat| by_category(cat) }
+        end
+
+        def find(key)
+          file_name = "#{key}#{self.extension}"
+
+          directory = select_directory(file_name)
+          directory ? new(File.join(category_directory(directory), file_name)) : nil
+        end
+
+        def categories
+          raise NotImplementedError
+        end
+
+        def extension
+          raise NotImplementedError
+        end
+
+        def base_dir
+          raise NotImplementedError
+        end
+
+        def by_category(category)
+          templates_for_directory(category_directory(category))
+        end
+
+        def category_directory(category)
+          File.join(base_dir, categories[category])
+        end
+
+        private
+
+        def select_directory(file_name)
+          categories.keys.find do |category|
+            File.exist?(File.join(category_directory(category), file_name))
+          end
+        end
+
+        def templates_for_directory(dir)
+          dir << '/' unless dir.end_with?('/')
+          Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) }
+        end
+
+        def filter_regex
+          @filter_reges ||= /#{Regexp.escape(extension)}\z/
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore.rb
new file mode 100644
index 00000000000..964fbfd4de3
--- /dev/null
+++ b/lib/gitlab/template/gitignore.rb
@@ -0,0 +1,22 @@
+module Gitlab
+  module Template
+    class Gitignore < BaseTemplate
+      class << self
+        def extension
+          '.gitignore'
+        end
+
+        def categories
+          {
+            "Languages" => '',
+            "Global"    => 'Global'
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/gitignore')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/gitlab_ci_yml.rb
new file mode 100644
index 00000000000..7f480fe33c0
--- /dev/null
+++ b/lib/gitlab/template/gitlab_ci_yml.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Template
+    class GitlabCiYml < BaseTemplate
+      def content
+        explanation = "# This file is a template, and might need editing before it works on your project."
+        [explanation, super].join("\n")
+      end
+
+      class << self
+        def extension
+          '.gitlab-ci.yml'
+        end
+
+        def categories
+          {
+            "General" => '',
+            "Pages" => 'Pages'
+          }
+        end
+
+        def base_dir
+          Rails.root.join('vendor/gitlab-ci-yml')
+        end
+      end
+    end
+  end
+end