Merge remote-tracking branch 'origin/master' into issue_14904

16 files changed, 289 insertions, 40 deletions

diff --git a/lib/api/api.rb b/lib/api/api.rb
index 7d65145176b..cc1004f8005 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -57,5 +57,6 @@ module API
     mount Builds
     mount Variables
     mount Runners
+    mount Licenses
   end
 end
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 60b9f5e0ece..716ca6f7ed9 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -439,5 +439,17 @@ module API
     class Variable < Grape::Entity
       expose :key, :value
     end
+
+    class RepoLicense < Grape::Entity
+      expose :key, :name, :nickname
+      expose :featured, as: :popular
+      expose :url, as: :html_url
+      expose(:source_url) { |license| license.meta['source'] }
+      expose(:description) { |license| license.meta['description'] }
+      expose(:conditions) { |license| license.meta['conditions'] }
+      expose(:permissions) { |license| license.meta['permissions'] }
+      expose(:limitations) { |license| license.meta['limitations'] }
+      expose :content
+    end
   end
 end
diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb
new file mode 100644
index 00000000000..187d2c04703
--- /dev/null
+++ b/lib/api/licenses.rb
@@ -0,0 +1,58 @@
+module API
+  # Licenses API
+  class Licenses < Grape::API
+    PROJECT_TEMPLATE_REGEX =
+    /[\<\{\[]
+      (project|description|
+      one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here
+    [\>\}\]]/xi.freeze
+    YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze
+    FULLNAME_TEMPLATE_REGEX =
+    /[\<\{\[]
+      (fullname|name\sof\s(author|copyright\sowner))
+    [\>\}\]]/xi.freeze
+
+    # Get the list of the available license templates
+    #
+    # Parameters:
+    #   popular - Filter licenses to only the popular ones
+    #
+    # Example Request:
+    #   GET /licenses
+    #   GET /licenses?popular=1
+    get 'licenses' do
+      options = {
+        featured: params[:popular].present? ? true : nil
+      }
+      present Licensee::License.all(options), with: Entities::RepoLicense
+    end
+
+    # Get text for specific license
+    #
+    # Parameters:
+    #   key (required) - The key of a license
+    #   project        - Copyrighted project name
+    #   fullname       - Full name of copyright holder
+    #
+    # Example Request:
+    #   GET /licenses/mit
+    #
+    get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do
+      required_attributes! [:key]
+
+      not_found!('License') unless Licensee::License.find(params[:key])
+
+      # We create a fresh Licensee::License object since we'll modify its
+      # content in place below.
+      license = Licensee::License.new(params[:key])
+
+      license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s)
+      license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present?
+
+      fullname = params[:fullname].presence || current_user.try(:name)
+      license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname
+
+      present license, with: Entities::RepoLicense
+    end
+  end
+end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index d1a10479e44..3e1ed3fe5c7 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -12,7 +12,7 @@ module API
       # Example Request:
       #   GET /projects/:id/repository/tags
       get ":id/repository/tags" do
-        present user_project.repo.tags.sort_by(&:name).reverse,
+        present user_project.repository.tags.sort_by(&:name).reverse,
                 with: Entities::RepoTag, project: user_project
       end
 
diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb
index 4e85d2c3c74..353c4ddebf8 100644
--- a/lib/ci/api/api.rb
+++ b/lib/ci/api/api.rb
@@ -23,6 +23,8 @@ module Ci
         rack_response({ 'message' => '500 Internal Server Error' }, 500)
       end
 
+      content_type :txt,  'text/plain'
+      content_type :json, 'application/json'
       format :json
 
       helpers ::Ci::API::Helpers
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 2e9a5d311f9..607359769d1 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -50,6 +50,39 @@ module Ci
           end
         end
 
+        # Send incremental log update - Runners only
+        #
+        # Parameters:
+        #   id (required) - The ID of a build
+        # Body:
+        #   content of logs to append
+        # Headers:
+        #   Content-Range (required) - range of content that was sent
+        #   BUILD-TOKEN (required) - The build authorization token
+        # Example Request:
+        #   PATCH /builds/:id/trace.txt
+        patch ":id/trace.txt" do
+          build = Ci::Build.find_by_id(params[:id])
+          not_found! unless build
+          authenticate_build_token!(build)
+          forbidden!('Build has been erased!') if build.erased?
+
+          error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+          content_range = request.headers['Content-Range']
+          content_range = content_range.split('-')
+
+          current_length = build.trace_length
+          unless current_length == content_range[0].to_i
+            return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+          end
+
+          build.append_trace(request.body.read, content_range[0].to_i)
+
+          status 202
+          header 'Build-Status', build.status
+          header 'Range', "0-#{build.trace_length}"
+        end
+
         # Authorize artifacts uploading for build - Runners only
         #
         # Parameters:
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index b8ede3a7edc..ff9887cba1e 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -4,12 +4,12 @@ module Ci
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
-    ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache]
+    ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache]
     ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services,
                         :allow_failure, :type, :stage, :when, :artifacts, :cache,
-                        :dependencies, :variables]
+                        :dependencies, :before_script, :after_script, :variables]
 
-    attr_reader :before_script, :image, :services, :path, :cache
+    attr_reader :before_script, :after_script, :image, :services, :path, :cache
 
     def initialize(config, path = nil)
       @config = YAML.safe_load(config, [Symbol], [], true)
@@ -55,6 +55,7 @@ module Ci
 
     def initial_parsing
       @before_script = @config[:before_script] || []
+      @after_script = @config[:after_script]
       @image = @config[:image]
       @services = @config[:services]
       @stages = @config[:stages] || @config[:types]
@@ -83,7 +84,7 @@ module Ci
       {
         stage_idx: stages.index(job[:stage]),
         stage: job[:stage],
-        commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}",
+        commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"),
         tag_list: job[:tags] || [],
         name: name,
         only: job[:only],
@@ -96,23 +97,32 @@ module Ci
           artifacts: job[:artifacts],
           cache: job[:cache] || @cache,
           dependencies: job[:dependencies],
+          after_script: job[:after_script] || @after_script,
         }.compact
       }
     end
 
-    def normalize_script(script)
-      if script.is_a? Array
-        script.join("\n")
-      else
-        script
+    def validate!
+      validate_global!
+
+      @jobs.each do |name, job|
+        validate_job!(name, job)
       end
+
+      true
     end
 
-    def validate!
+    private
+
+    def validate_global!
       unless validate_array_of_strings(@before_script)
         raise ValidationError, "before_script should be an array of strings"
       end
 
+      unless @after_script.nil? || validate_array_of_strings(@after_script)
+        raise ValidationError, "after_script should be an array of strings"
+      end
+
       unless @image.nil? || @image.is_a?(String)
         raise ValidationError, "image should be a string"
       end
@@ -129,31 +139,28 @@ module Ci
         raise ValidationError, "variables should be a map of key-value strings"
       end
 
-      if @cache
-        if @cache[:key] && !validate_string(@cache[:key])
-          raise ValidationError, "cache:key parameter should be a string"
-        end
-
-        if @cache[:untracked] && !validate_boolean(@cache[:untracked])
-          raise ValidationError, "cache:untracked parameter should be an boolean"
-        end
+      validate_global_cache! if @cache
+    end
 
-        if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
-          raise ValidationError, "cache:paths parameter should be an array of strings"
-        end
+    def validate_global_cache!
+      if @cache[:key] && !validate_string(@cache[:key])
+        raise ValidationError, "cache:key parameter should be a string"
       end
 
-      @jobs.each do |name, job|
-        validate_job!(name, job)
+      if @cache[:untracked] && !validate_boolean(@cache[:untracked])
+        raise ValidationError, "cache:untracked parameter should be an boolean"
       end
 
-      true
+      if @cache[:paths] && !validate_array_of_strings(@cache[:paths])
+        raise ValidationError, "cache:paths parameter should be an array of strings"
+      end
     end
 
     def validate_job!(name, job)
       validate_job_name!(name)
       validate_job_keys!(name, job)
       validate_job_types!(name, job)
+      validate_job_script!(name, job)
 
       validate_job_stage!(name, job) if job[:stage]
       validate_job_variables!(name, job) if job[:variables]
@@ -162,8 +169,6 @@ module Ci
       validate_job_dependencies!(name, job) if job[:dependencies]
     end
 
-    private
-
     def validate_job_name!(name)
       if name.blank? || !validate_string(name)
         raise ValidationError, "job name should be non-empty string"
@@ -179,10 +184,6 @@ module Ci
     end
 
     def validate_job_types!(name, job)
-      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
-        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
-      end
-
       if job[:image] && !validate_string(job[:image])
         raise ValidationError, "#{name} job: image should be a string"
       end
@@ -212,6 +213,20 @@ module Ci
       end
     end
 
+    def validate_job_script!(name, job)
+      if !validate_string(job[:script]) && !validate_array_of_strings(job[:script])
+        raise ValidationError, "#{name} job: script should be a string or an array of a strings"
+      end
+
+      if job[:before_script] && !validate_array_of_strings(job[:before_script])
+        raise ValidationError, "#{name} job: before_script should be an array of strings"
+      end
+
+      if job[:after_script] && !validate_array_of_strings(job[:after_script])
+        raise ValidationError, "#{name} job: after_script should be an array of strings"
+      end
+    end
+
     def validate_job_stage!(name, job)
       unless job[:stage].is_a?(String) && job[:stage].in?(stages)
         raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}"
diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb
index 2eae55e534b..440dd44ece7 100644
--- a/lib/file_size_validator.rb
+++ b/lib/file_size_validator.rb
@@ -1,9 +1,9 @@
 class FileSizeValidator < ActiveModel::EachValidator
-  MESSAGES  = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze
-  CHECKS    = { is: :==, minimum: :>=, maximum: :<= }.freeze
+  MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze
+  CHECKS   = { is: :==, minimum: :>=, maximum: :<= }.freeze
 
-  DEFAULT_TOKENIZER = lambda { |value| value.split(//) }
-  RESERVED_OPTIONS  = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long]
+  DEFAULT_TOKENIZER = -> (value) { value.split(//) }.freeze
+  RESERVED_OPTIONS  = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long].freeze
 
   def initialize(options)
     if range = (options.delete(:in) || options.delete(:within))
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 0b1ed510229..0f9e3ee14ee 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -16,7 +16,8 @@ module Gitlab
       end
 
       def execute
-        import_issues && import_pull_requests && import_wiki
+        import_labels && import_milestones && import_issues &&
+          import_pull_requests && import_wiki
       end
 
       private
@@ -25,6 +26,26 @@ module Gitlab
         @import_data_credentials ||= project.import_data.credentials if project.import_data
       end
 
+      def import_labels
+        client.labels(project.import_source).each do |raw_data|
+          Label.create!(LabelFormatter.new(project, raw_data).attributes)
+        end
+
+        true
+      rescue ActiveRecord::RecordInvalid => e
+        raise Projects::ImportService::Error, e.message
+      end
+
+      def import_milestones
+        client.list_milestones(project.import_source, state: :all).each do |raw_data|
+          Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
+        end
+
+        true
+      rescue ActiveRecord::RecordInvalid => e
+        raise Projects::ImportService::Error, e.message
+      end
+
       def import_issues
         client.list_issues(project.import_source, state: :all,
                                                   sort: :created,
@@ -33,6 +54,7 @@ module Gitlab
 
           if gh_issue.valid?
             issue = Issue.create!(gh_issue.attributes)
+            apply_labels(gh_issue.number, issue)
 
             if gh_issue.has_comments?
               import_comments(gh_issue.number, issue)
@@ -55,6 +77,7 @@ module Gitlab
             merge_request = MergeRequest.new(pull_request.attributes)
 
             if merge_request.save
+              apply_labels(pull_request.number, merge_request)
               import_comments(pull_request.number, merge_request)
               import_comments_on_diff(pull_request.number, merge_request)
             end
@@ -66,6 +89,18 @@ module Gitlab
         raise Projects::ImportService::Error, e.message
       end
 
+      def apply_labels(number, issuable)
+        issue = client.issue(project.import_source, number)
+
+        if issue.labels.count > 0
+          label_ids = issue.labels.map do |raw|
+            Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id)
+          end
+
+          issuable.update_attribute(:label_ids, label_ids)
+        end
+      end
+
       def import_comments(issue_number, noteable)
         comments = client.issue_comments(project.import_source, issue_number)
         create_comments(comments, noteable)
diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb
index acb332cb0cb..c8173913b4e 100644
--- a/lib/gitlab/github_import/issue_formatter.rb
+++ b/lib/gitlab/github_import/issue_formatter.rb
@@ -5,6 +5,7 @@ module Gitlab
         {
           iid: number,
           project: project,
+          milestone: milestone,
           title: raw_data.title,
           description: description,
           state: state,
@@ -55,6 +56,12 @@ module Gitlab
         @formatter.author_line(author) + body
       end
 
+      def milestone
+        if raw_data.milestone.present?
+          project.milestones.find_by(iid: raw_data.milestone.number)
+        end
+      end
+
       def state
         raw_data.state == 'closed' ? 'closed' : 'opened'
       end
diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb
new file mode 100644
index 00000000000..c2b9d40b511
--- /dev/null
+++ b/lib/gitlab/github_import/label_formatter.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  module GithubImport
+    class LabelFormatter < BaseFormatter
+      def attributes
+        {
+          project: project,
+          title: title,
+          color: color
+        }
+      end
+
+      private
+
+      def color
+        "##{raw_data.color}"
+      end
+
+      def title
+        raw_data.name
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb
new file mode 100644
index 00000000000..e91a7e328cf
--- /dev/null
+++ b/lib/gitlab/github_import/milestone_formatter.rb
@@ -0,0 +1,48 @@
+module Gitlab
+  module GithubImport
+    class MilestoneFormatter < BaseFormatter
+      def attributes
+        {
+          iid: number,
+          project: project,
+          title: title,
+          description: description,
+          due_date: due_date,
+          state: state,
+          created_at: created_at,
+          updated_at: updated_at
+        }
+      end
+
+      private
+
+      def number
+        raw_data.number
+      end
+
+      def title
+        raw_data.title
+      end
+
+      def description
+        raw_data.description
+      end
+
+      def due_date
+        raw_data.due_on
+      end
+
+      def state
+        raw_data.state == 'closed' ? 'closed' : 'active'
+      end
+
+      def created_at
+        raw_data.created_at
+      end
+
+      def updated_at
+        state == 'closed' ? raw_data.closed_at : raw_data.updated_at
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb
index 5ee8a14624a..d21b942ad4b 100644
--- a/lib/gitlab/github_import/pull_request_formatter.rb
+++ b/lib/gitlab/github_import/pull_request_formatter.rb
@@ -11,6 +11,7 @@ module Gitlab
           target_project: target_project,
           target_branch: target_branch.name,
           state: state,
+          milestone: milestone,
           author_id: author_id,
           assignee_id: assignee_id,
           created_at: raw_data.created_at,
@@ -58,6 +59,12 @@ module Gitlab
         formatter.author_line(author) + body
       end
 
+      def milestone
+        if raw_data.milestone.present?
+          project.milestones.find_by(iid: raw_data.milestone.number)
+        end
+      end
+
       def source_project
         project
       end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 484970c5a10..f82dce14865 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -14,7 +14,8 @@ module Gitlab
         method_call_threshold: current_application_settings[:metrics_method_call_threshold],
         host:                  current_application_settings[:metrics_host],
         port:                  current_application_settings[:metrics_port],
-        sample_interval:       current_application_settings[:metrics_sample_interval] || 15
+        sample_interval:       current_application_settings[:metrics_sample_interval] || 15,
+        packet_size:           current_application_settings[:metrics_packet_size] || 1
       }
     end
 
@@ -41,9 +42,9 @@ module Gitlab
       prepared = prepare_metrics(metrics)
 
       pool.with do |connection|
-        prepared.each do |metric|
+        prepared.each_slice(settings[:packet_size]) do |slice|
           begin
-            connection.write_points([metric])
+            connection.write_points(slice)
           rescue StandardError
           end
         end
diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb
index 97d1edab9c1..67622f321a6 100644
--- a/lib/gitlab/push_data_builder.rb
+++ b/lib/gitlab/push_data_builder.rb
@@ -36,11 +36,12 @@ module Gitlab
           commit.hook_attrs(with_changed_files: true)
         end
 
-        type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push"
+        type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push'
 
         # Hash to be passed as post_receive_data
         data = {
           object_kind: type,
+          event_name: type,
           before: oldrev,
           after: newrev,
           ref: ref,
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index f1943222edf..2bbbd3074e8 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -20,6 +20,8 @@ module Gitlab
         merge_request_url(object)
       when Note
         note_url
+      when WikiPage
+        wiki_page_url
       else
         raise NotImplementedError.new("No URL builder defined for #{object.class}")
       end
@@ -58,5 +60,9 @@ module Gitlab
         project_snippet_url(snippet, anchor: dom_id(object))
       end
     end
+
+    def wiki_page_url
+      "#{Gitlab.config.gitlab.url}#{object.wiki.wiki_base_path}/#{object.slug}"
+    end
   end
 end