Merge branch 'master' of dev.gitlab.org:gitlab/gitlabhq into backup-archive-permissions

13 files changed, 300 insertions, 95 deletions

diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index aa43e1dffd9..ce21c699e8f 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -229,7 +229,7 @@ module API
 
         authorize! :read_merge_request, merge_request
 
-        present paginate(merge_request.notes), with: Entities::MRNote
+        present paginate(merge_request.notes.fresh), with: Entities::MRNote
       end
 
       # Post comment to merge request
diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb
index 03cef30c97d..12292f614e9 100644
--- a/lib/gitlab/backend/grack_auth.rb
+++ b/lib/gitlab/backend/grack_auth.rb
@@ -26,7 +26,12 @@ module Grack
       auth!
 
       if project && authorized_request?
-        @app.call(env)
+        if ENV['GITLAB_GRACK_AUTH_ONLY'] == '1'
+          # Tell gitlab-git-http-server the request is OK, and what the GL_ID is
+          render_grack_auth_ok
+        else
+          @app.call(env)
+        end
       elsif @user.nil? && !@gitlab_ci
         unauthorized
       else
@@ -174,6 +179,10 @@ module Grack
       end
     end
 
+    def render_grack_auth_ok
+      [200, { "Content-Type" => "application/json" }, [JSON.dump({ 'GL_ID' => Gitlab::ShellEnv.gl_id(@user) })]]
+    end
+
     def render_not_found
       [404, { "Content-Type" => "text/plain" }, ["Not Found"]]
     end
diff --git a/lib/gitlab/backend/shell_env.rb b/lib/gitlab/backend/shell_env.rb
index 17ec029eed4..9f5adee594a 100644
--- a/lib/gitlab/backend/shell_env.rb
+++ b/lib/gitlab/backend/shell_env.rb
@@ -7,7 +7,7 @@ module Gitlab
     def set_env(user)
       # Set GL_ID env variable
       if user
-        ENV['GL_ID'] = "user-#{user.id}"
+        ENV['GL_ID'] = gl_id(user)
       end
     end
 
@@ -15,5 +15,14 @@ module Gitlab
       # Reset GL_ID env variable
       ENV['GL_ID'] = nil
     end
+
+    def gl_id(user)
+      if user.present?
+        "user-#{user.id}"
+      else
+        # This empty string is used in the render_grack_auth_ok method
+        ""
+      end
+    end
   end
 end
diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb
index 5b1952b9675..778b76f6890 100644
--- a/lib/gitlab/bitbucket_import/client.rb
+++ b/lib/gitlab/bitbucket_import/client.rb
@@ -1,6 +1,8 @@
 module Gitlab
   module BitbucketImport
     class Client
+      class Unauthorized < StandardError; end
+
       attr_reader :consumer, :api
 
       def initialize(access_token = nil, access_token_secret = nil)
@@ -46,23 +48,23 @@ module Gitlab
       end
 
       def user
-        JSON.parse(api.get("/api/1.0/user").body)
+        JSON.parse(get("/api/1.0/user").body)
       end
 
       def issues(project_identifier)
-        JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues").body)
+        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body)
       end
 
       def issue_comments(project_identifier, issue_id)
-        JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
+        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
       end
 
       def project(project_identifier)
-        JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}").body)
+        JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body)
       end
 
       def find_deploy_key(project_identifier, key)
-        JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
+        JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key|
           deploy_key["key"].chomp == key.chomp
         end
       end
@@ -82,11 +84,18 @@ module Gitlab
       end
 
       def projects
-        JSON.parse(api.get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
+        JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" }
       end
 
       private
 
+      def get(url)
+        response = api.get(url)
+        raise Unauthorized if (400..499).include?(response.code.to_i)
+
+        response
+      end
+
       def config
         Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
       end
diff --git a/lib/gitlab/o_auth/provider.rb b/lib/gitlab/o_auth/provider.rb
index f986499a27c..90c3fe8da33 100644
--- a/lib/gitlab/o_auth/provider.rb
+++ b/lib/gitlab/o_auth/provider.rb
@@ -1,18 +1,30 @@
 module Gitlab
   module OAuth
     class Provider
-      def self.names
-        providers = []
+      def self.providers
+        Devise.omniauth_providers
+      end
 
-        Gitlab.config.ldap.servers.values.each do |server|
-          providers << server['provider_name']
-        end
+      def self.enabled?(name)
+        providers.include?(name.to_sym)
+      end
 
-        Gitlab.config.omniauth.providers.each do |provider|
-          providers << provider['name']
+      def self.ldap_provider?(name)
+        name.to_s.start_with?('ldap')
+      end
+
+      def self.config_for(name)
+        name = name.to_s
+        if ldap_provider?(name)
+          Gitlab::LDAP::Config.new(name).options
+        else
+          Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
         end
+      end
 
-        providers
+      def self.label_for(name)
+        config = config_for(name)
+        (config && config['label']) || name.to_s.titleize
       end
     end
   end
diff --git a/lib/gitlab/satellite/merge_action.rb b/lib/gitlab/satellite/merge_action.rb
index 1f2e5f82dd5..f9bf286697e 100644
--- a/lib/gitlab/satellite/merge_action.rb
+++ b/lib/gitlab/satellite/merge_action.rb
@@ -33,7 +33,7 @@ module Gitlab
             merge_repo.git.push(default_options, :origin, merge_request.target_branch)
 
             # remove source branch
-            if merge_request.should_remove_source_branch && !project.root_ref?(merge_request.source_branch) && !merge_request.for_fork?
+            if merge_request.remove_source_branch?
               # will raise CommandFailed when push fails
               merge_repo.git.push(default_options, :origin, ":#{merge_request.source_branch}")
             end
diff --git a/lib/redcarpet/render/gitlab_html.rb b/lib/redcarpet/render/gitlab_html.rb
index 2f7aff03c2a..04440e4f68d 100644
--- a/lib/redcarpet/render/gitlab_html.rb
+++ b/lib/redcarpet/render/gitlab_html.rb
@@ -22,10 +22,10 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML
     ERB::Util.html_escape_once(text)
   end
 
-  # Stolen from Rugments::Plugins::Redcarpet as this module is not required
-  # from Rugments's gem root.
+  # Stolen from Rouge::Plugins::Redcarpet as this module is not required
+  # from Rouge's gem root.
   def block_code(code, language)
-    lexer = Rugments::Lexer.find_fancy(language, code) || Rugments::Lexers::PlainText
+    lexer = Rouge::Lexer.find_fancy(language, code) || Rouge::Lexers::PlainText
 
     # XXX HACK: Redcarpet strips hard tabs out of code blocks,
     # so we assume you're not using leading spaces that aren't tabs,
@@ -34,7 +34,7 @@ class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML
       code.gsub!(/^    /, "\t")
     end
 
-    formatter = Rugments::Formatters::HTML.new(
+    formatter = Rouge::Formatters::HTMLGitlab.new(
       cssclass: "code highlight #{@color_scheme} #{lexer.tag}"
     )
     formatter.format(lexer.lex(code))
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
new file mode 100644
index 00000000000..485af6832d7
--- /dev/null
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -0,0 +1,168 @@
+require 'cgi'
+
+module Rouge
+  module Formatters
+    class HTMLGitlab < Rouge::Formatter
+      tag 'html_gitlab'
+
+      # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance.
+      #
+      # [+nowrap+]          If set to True, don't wrap the output at all, not
+      #                     even inside a <tt><pre></tt> tag (default: false).
+      # [+cssclass+]        CSS class for the wrapping <tt><div></tt> tag
+      #                     (default: 'highlight').
+      # [+linenos+]         If set to 'table', output line numbers as a table
+      #                     with two cells, one containing the line numbers,
+      #                     the other the whole code. This is copy paste friendly,
+      #                     but may cause alignment problems with some browsers
+      #                     or fonts. If set to 'inline', the line numbers will
+      #                     be integrated in the <tt><pre></tt> tag that contains
+      #                     the code (default: nil).
+      # [+linenostart+]     The line number for the first line (default: 1).
+      # [+lineanchors+]     If set to true the formatter will wrap each output
+      #                     line in an anchor tag with a name of L-linenumber.
+      #                     This allows easy linking to certain lines
+      #                     (default: false).
+      # [+lineanchorsid+]   If lineanchors is true the name of the anchors can
+      #                     be changed with lineanchorsid to e.g. foo-linenumber
+      #                     (default: 'L').
+      # [+anchorlinenos+]   If set to true, will wrap line numbers in <tt><a></tt>
+      #                     tags. Used in combination with linenos and lineanchors
+      #                     (default: false).
+      # [+inline_theme+]    Inline CSS styles for the <pre> tag (default: false).
+      def initialize(
+          nowrap: false,
+          cssclass: 'highlight',
+          linenos: nil,
+          linenostart: 1,
+          lineanchors: false,
+          lineanchorsid: 'L',
+          anchorlinenos: false,
+          inline_theme: nil
+        )
+        @nowrap = nowrap
+        @cssclass = cssclass
+        @linenos = linenos
+        @linenostart = linenostart
+        @lineanchors = lineanchors
+        @lineanchorsid = lineanchorsid
+        @anchorlinenos = anchorlinenos
+        @inline_theme = Theme.find(@inline_theme).new if @inline_theme.is_a?(String)
+      end
+
+      def render(tokens)
+        case @linenos
+        when 'table'
+          render_tableized(tokens)
+        when 'inline'
+          render_untableized(tokens)
+        else
+          render_untableized(tokens)
+        end
+      end
+
+      alias_method :format, :render
+
+      private
+
+      def render_untableized(tokens)
+        data = process_tokens(tokens)
+
+        html = ''
+        html << "<pre class=\"#{@cssclass}\"><code>" unless @nowrap
+        html << wrap_lines(data[:code])
+        html << "</code></pre>\n" unless @nowrap
+        html
+      end
+
+      def render_tableized(tokens)
+        data = process_tokens(tokens)
+
+        html = ''
+        html << "<div class=\"#{@cssclass}\">" unless @nowrap
+        html << '<table><tbody>'
+        html << "<td class=\"linenos\"><pre>"
+        html << wrap_linenos(data[:numbers])
+        html << '</pre></td>'
+        html << "<td class=\"lines\"><pre><code>"
+        html << wrap_lines(data[:code])
+        html << '</code></pre></td>'
+        html << '</tbody></table>'
+        html << '</div>' unless @nowrap
+        html
+      end
+
+      def process_tokens(tokens)
+        num_lines = 0
+        last_val = ''
+        rendered = ''
+
+        tokens.each do |tok, val|
+          last_val = val
+          num_lines += val.scan(/\n/).size
+          rendered << span(tok, val)
+        end
+
+        numbers = (@linenostart..num_lines + @linenostart - 1).to_a
+
+        { numbers: numbers, code: rendered }
+      end
+
+      def wrap_linenos(numbers)
+        if @anchorlinenos
+          numbers.map! do |number|
+            "<a href=\"##{@lineanchorsid}#{number}\">#{number}</a>"
+          end
+        end
+        numbers.join("\n")
+      end
+
+      def wrap_lines(rendered)
+        if @lineanchors
+          lines = rendered.split("\n")
+          lines = lines.each_with_index.map do |line, index|
+            number = index + @linenostart
+
+            if @linenos == 'inline'
+              "<a name=\"L#{number}\"></a>" \
+              "<span class=\"linenos\">#{number}</span>" \
+              "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
+              '</span>'
+            else
+              "<span id=\"#{@lineanchorsid}#{number}\" class=\"line\">#{line}" \
+              '</span>'
+            end
+          end
+          lines.join("\n")
+        else
+          if @linenos == 'inline'
+            lines = rendered.split("\n")
+            lines = lines.each_with_index.map do |line, index|
+              number = index + @linenostart
+              "<span class=\"linenos\">#{number}</span>#{line}"
+            end
+            lines.join("\n")
+          else
+            rendered
+          end
+        end
+      end
+
+      def span(tok, val)
+        # http://stackoverflow.com/a/1600584/2587286
+        val = CGI.escapeHTML(val)
+
+        if tok.shortname.empty?
+          val
+        else
+          if @inline_theme
+            rules = @inline_theme.style_for(tok).rendered_rules
+            "<span style=\"#{rules.to_a.join(';')}\">#{val}</span>"
+          else
+            "<span class=\"#{tok.shortname}\">#{val}</span>"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index edb987875df..efa0898900f 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -38,6 +38,11 @@ upstream gitlab {
   server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0;
 }
 
+## Experimental: gitlab-git-http-server
+# upstream gitlab-git-http-server {
+#   server localhost:8181;
+# }
+
 ## Normal HTTP host
 server {
   ## Either remove "default_server" from the listen line below, 
@@ -109,6 +114,26 @@ server {
     proxy_pass http://gitlab;
   }
 
+  ## Experimental: send Git HTTP traffic to gitlab-git-http-server instead of Unicorn
+  # location ~ [-\/\w\.]+\.git\/ {
+  #   ## If you use HTTPS make sure you disable gzip compression
+  #   ## to be safe against BREACH attack.
+  #   # gzip off;
+
+  #   ## https://github.com/gitlabhq/gitlabhq/issues/694
+  #   ## Some requests take more than 30 seconds.
+  #   proxy_read_timeout      300;
+  #   proxy_connect_timeout   300;
+  #   proxy_redirect          off;
+
+  #   proxy_set_header    Host                $http_host;
+  #   proxy_set_header    X-Real-IP           $remote_addr;
+  #   proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+  #   proxy_set_header    X-Forwarded-Proto   $scheme;
+
+  #   proxy_pass http://gitlab-git-http-server;
+  # }
+
   ## Enable gzip compression as per rails guide:
   ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression
   ## WARNING: If you are using relative urls remove the block below
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index 766559b49f6..314525518f1 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -42,6 +42,11 @@ upstream gitlab {
   server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0;
 }
 
+## Experimental: gitlab-git-http-server
+# upstream gitlab-git-http-server {
+#   server localhost:8181;
+# }
+
 ## Redirects all HTTP traffic to the HTTPS host
 server {
   ## Either remove "default_server" from the listen line below, 
@@ -156,6 +161,26 @@ server {
     proxy_pass http://gitlab;
   }
 
+  ## Experimental: send Git HTTP traffic to gitlab-git-http-server instead of Unicorn
+  # location ~ [-\/\w\.]+\.git\/ {
+  #   ## If you use HTTPS make sure you disable gzip compression
+  #   ## to be safe against BREACH attack.
+  #   gzip off;
+
+  #   ## https://github.com/gitlabhq/gitlabhq/issues/694
+  #   ## Some requests take more than 30 seconds.
+  #   proxy_read_timeout      300;
+  #   proxy_connect_timeout   300;
+  #   proxy_redirect          off;
+
+  #   proxy_set_header    Host                $http_host;
+  #   proxy_set_header    X-Real-IP           $remote_addr;
+  #   proxy_set_header    X-Forwarded-Ssl     on;
+  #   proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+  #   proxy_set_header    X-Forwarded-Proto   $scheme;
+  #   proxy_pass http://gitlab-git-http-server;
+  # }
+
   ## Enable gzip compression as per rails guide:
   ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression
   ## WARNING: If you are using relative urls remove the block below
diff --git a/lib/tasks/gitlab/mail_google_schema_whitelisting.rake b/lib/tasks/gitlab/mail_google_schema_whitelisting.rake
deleted file mode 100644
index 102c6ae55d5..00000000000
--- a/lib/tasks/gitlab/mail_google_schema_whitelisting.rake
+++ /dev/null
@@ -1,73 +0,0 @@
-require "#{Rails.root}/app/helpers/emails_helper"
-require 'action_view/helpers'
-extend ActionView::Helpers
-
-include ActionView::Context
-include EmailsHelper
-
-namespace :gitlab do
-  desc "Email google whitelisting email with example email for actions in inbox"
-  task mail_google_schema_whitelisting: :environment do
-    subject = "Rails | Implemented feature"
-    url = "#{Gitlab.config.gitlab.url}/base/rails-project/issues/#{rand(1..100)}#note_#{rand(10..1000)}"
-    schema = email_action(url)
-    body = email_template(schema, url)
-    mail = Notify.test_email("schema.whitelisting+sample@gmail.com", subject, body.html_safe)
-    if send_now
-      mail.deliver
-    else
-      puts "WOULD SEND:"
-    end
-    puts mail
-  end
-
-  def email_template(schema, url)
-    "<html lang='en'>
-      <head>
-        <meta content='text/html; charset=utf-8' http-equiv='Content-Type'>
-          <title>
-            GitLab
-          </title>
-        </meta>
-      </head>
-      <style>
-        img {
-          max-width: 100%;
-          height: auto;
-        }
-        p.details {
-          font-style:italic;
-          color:#777
-        }
-        .footer p {
-          font-size:small;
-          color:#777
-        }
-      </style>
-      <body>
-        <div class='content'>
-          <div>
-           <p>I like it :+1: </p>
-          </div>
-        </div>
-
-        <div class='footer' style='margin-top: 10px;'>
-          <p>
-          <br>
-            <a href=\"#{url}\">View it on GitLab</a>
-            You're receiving this notification because you are a member of the Base / Rails Project project team.
-            #{schema}
-          </p>
-        </div>
-      </body>
-    </html>"
-  end
-
-  def send_now
-    if ENV['SEND'] == "true"
-      true
-    else
-      false
-    end
-  end
-end
diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake
new file mode 100644
index 00000000000..9b636f12d9f
--- /dev/null
+++ b/lib/tasks/gitlab/update_commit_count.rake
@@ -0,0 +1,20 @@
+namespace :gitlab do
+  desc "GitLab | Update commit count for projects"
+  task update_commit_count: :environment do
+    projects = Project.where(commit_count: 0)
+    puts "#{projects.size} projects need to be updated. This might take a while."
+    ask_to_continue unless ENV['force'] == 'yes'
+
+    projects.find_each(batch_size: 100) do |project|
+      print "#{project.name_with_namespace.yellow} ... "
+
+      unless project.repo_exists?
+        puts "skipping, because the repo is empty".magenta
+        next
+      end
+
+      project.update_commit_count
+      puts project.commit_count.to_s.green
+    end
+  end
+end
diff --git a/lib/unfold_form.rb b/lib/unfold_form.rb
index 46b12beeaaf..fcd01503d1b 100644
--- a/lib/unfold_form.rb
+++ b/lib/unfold_form.rb
@@ -8,4 +8,5 @@ class UnfoldForm
   attribute :bottom, Boolean
   attribute :unfold, Boolean, default: true
   attribute :offset, Integer
+  attribute :indent, Integer, default: 0
 end