Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into fix/sidekiq-mem-killer-debug

18 files changed, 374 insertions, 78 deletions

diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 4c43257c48a..8a03a41e9c5 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -59,6 +59,41 @@ module API
       end
     end
 
+    resource :groups do
+      # Get a list of group issues
+      #
+      # Parameters:
+      #   id (required) - The ID of a group
+      #   state (optional) - Return "opened" or "closed" issues
+      #   labels (optional) - Comma-separated list of label names
+      #   milestone (optional) - Milestone title
+      #   order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at`
+      #   sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc`
+      #
+      # Example Requests:
+      #   GET /groups/:id/issues
+      #   GET /groups/:id/issues?state=opened
+      #   GET /groups/:id/issues?state=closed
+      #   GET /groups/:id/issues?labels=foo
+      #   GET /groups/:id/issues?labels=foo,bar
+      #   GET /groups/:id/issues?labels=foo,bar&state=opened
+      #   GET /groups/:id/issues?milestone=1.0.0
+      #   GET /groups/:id/issues?milestone=1.0.0&state=closed
+      get ":id/issues" do
+        group = find_group(params[:id])
+
+        params[:state] ||= 'opened'
+        params[:group_id] = group.id
+        params[:milestone_title] = params.delete(:milestone)
+        params[:label_name] = params.delete(:labels)
+        params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort]
+
+        issues = IssuesFinder.new(current_user, params).execute
+
+        present paginate(issues), with: Entities::Issue, current_user: current_user
+      end
+    end
+
     resource :projects do
       # Get a list of project issues
       #
diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb
index c753a84a20d..c59a80dd1c7 100644
--- a/lib/banzai/filter/redactor_filter.rb
+++ b/lib/banzai/filter/redactor_filter.rb
@@ -7,40 +7,13 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        nodes = Querying.css(doc, 'a.gfm[data-reference-type]')
-        visible = nodes_visible_to_user(nodes)
-
-        nodes.each do |node|
-          unless visible.include?(node)
-            # The reference should be replaced by the original text,
-            # which is not always the same as the rendered text.
-            text = node.attr('data-original') || node.text
-            node.replace(text)
-          end
-        end
+        Redactor.new(project, current_user).redact([doc])
 
         doc
       end
 
       private
 
-      def nodes_visible_to_user(nodes)
-        per_type = Hash.new { |h, k| h[k] = [] }
-        visible = Set.new
-
-        nodes.each do |node|
-          per_type[node.attr('data-reference-type')] << node
-        end
-
-        per_type.each do |type, nodes|
-          parser = Banzai::ReferenceParser[type].new(project, current_user)
-
-          visible.merge(parser.nodes_visible_to_user(current_user, nodes))
-        end
-
-        visible
-      end
-
       def current_user
         context[:current_user]
       end
diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb
new file mode 100644
index 00000000000..bab6a9934d1
--- /dev/null
+++ b/lib/banzai/note_renderer.rb
@@ -0,0 +1,22 @@
+module Banzai
+  module NoteRenderer
+    # Renders a collection of Note instances.
+    #
+    # notes - The notes to render.
+    # project - The project to use for rendering/redacting.
+    # user - The user viewing the notes.
+    # path - The request path.
+    # wiki - The project's wiki.
+    # git_ref - The current Git reference.
+    def self.render(notes, project, user = nil, path = nil, wiki = nil, git_ref = nil)
+      renderer = ObjectRenderer.new(project,
+                                    user,
+                                    requested_path: path,
+                                    project_wiki: wiki,
+                                    ref: git_ref,
+                                    pipeline: :note)
+
+      renderer.render(notes, :note)
+    end
+  end
+end
diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb
new file mode 100644
index 00000000000..f0e4f28bf12
--- /dev/null
+++ b/lib/banzai/object_renderer.rb
@@ -0,0 +1,85 @@
+module Banzai
+  # Class for rendering multiple objects (e.g. Note instances) in a single pass.
+  #
+  # Rendered Markdown is stored in an attribute in every object based on the
+  # name of the attribute containing the Markdown. For example, when the
+  # attribute `note` is rendered the HTML is stored in `note_html`.
+  class ObjectRenderer
+    attr_reader :project, :user
+
+    # Make sure to set the appropriate pipeline in the `raw_context` attribute
+    # (e.g. `:note` for Note instances).
+    #
+    # project - A Project to use for rendering and redacting Markdown.
+    # user - The user viewing the Markdown/HTML documents, if any.
+    # context - A Hash containing extra attributes to use in the rendering
+    #           pipeline.
+    def initialize(project, user = nil, raw_context = {})
+      @project = project
+      @user = user
+      @raw_context = raw_context
+    end
+
+    # Renders and redacts an Array of objects.
+    #
+    # objects - The objects to render
+    # attribute - The attribute containing the raw Markdown to render.
+    #
+    # Returns the same input objects.
+    def render(objects, attribute)
+      documents = render_objects(objects, attribute)
+      redacted = redact_documents(documents)
+
+      objects.each_with_index do |object, index|
+        object.__send__("#{attribute}_html=", redacted.fetch(index))
+      end
+
+      objects
+    end
+
+    # Renders the attribute of every given object.
+    def render_objects(objects, attribute)
+      objects.map do |object|
+        render_attribute(object, attribute)
+      end
+    end
+
+    # Redacts the list of documents.
+    #
+    # Returns an Array containing the redacted documents.
+    def redact_documents(documents)
+      redactor = Redactor.new(project, user)
+
+      redactor.redact(documents).map do |document|
+        document.to_html.html_safe
+      end
+    end
+
+    # Returns a Banzai context for the given object and attribute.
+    def context_for(object, attribute)
+      context = base_context.merge(cache_key: [object, attribute])
+
+      if object.respond_to?(:author)
+        context[:author] = object.author
+      end
+
+      context
+    end
+
+    # Renders the attribute of an object.
+    #
+    # Returns a `Nokogiri::HTML::Document`.
+    def render_attribute(object, attribute)
+      context = context_for(object, attribute)
+
+      string = object.__send__(attribute)
+      html = Banzai.render(string, context)
+
+      Banzai::Pipeline[:relative_link].to_document(html, context)
+    end
+
+    def base_context
+      @base_context ||= @raw_context.merge(current_user: user, project: project)
+    end
+  end
+end
diff --git a/lib/banzai/pipeline/relative_link_pipeline.rb b/lib/banzai/pipeline/relative_link_pipeline.rb
new file mode 100644
index 00000000000..270990e7ab4
--- /dev/null
+++ b/lib/banzai/pipeline/relative_link_pipeline.rb
@@ -0,0 +1,11 @@
+module Banzai
+  module Pipeline
+    class RelativeLinkPipeline < BasePipeline
+      def self.filters
+        FilterArray[
+          Filter::RelativeLinkFilter
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb
new file mode 100644
index 00000000000..ffd267d5e9a
--- /dev/null
+++ b/lib/banzai/redactor.rb
@@ -0,0 +1,69 @@
+module Banzai
+  # Class for removing Markdown references a certain user is not allowed to
+  # view.
+  class Redactor
+    attr_reader :user, :project
+
+    # project - A Project to use for redacting links.
+    # user - The currently logged in user (if any).
+    def initialize(project, user = nil)
+      @project = project
+      @user = user
+    end
+
+    # Redacts the references in the given Array of documents.
+    #
+    # This method modifies the given documents in-place.
+    #
+    # documents - A list of HTML documents containing references to redact.
+    #
+    # Returns the documents passed as the first argument.
+    def redact(documents)
+      nodes = documents.flat_map do |document|
+        Querying.css(document, 'a.gfm[data-reference-type]')
+      end
+
+      redact_nodes(nodes)
+
+      documents
+    end
+
+    # Redacts the given nodes
+    #
+    # nodes - An Array of HTML nodes to redact.
+    def redact_nodes(nodes)
+      visible = nodes_visible_to_user(nodes)
+
+      nodes.each do |node|
+        unless visible.include?(node)
+          # The reference should be replaced by the original text,
+          # which is not always the same as the rendered text.
+          text = node.attr('data-original') || node.text
+          node.replace(text)
+        end
+      end
+    end
+
+    # Returns the nodes visible to the current user.
+    #
+    # nodes - The input nodes to check.
+    #
+    # Returns a new Array containing the visible nodes.
+    def nodes_visible_to_user(nodes)
+      per_type = Hash.new { |h, k| h[k] = [] }
+      visible = Set.new
+
+      nodes.each do |node|
+        per_type[node.attr('data-reference-type')] << node
+      end
+
+      per_type.each do |type, nodes|
+        parser = Banzai::ReferenceParser[type].new(project, user)
+
+        visible.merge(parser.nodes_visible_to_user(user, nodes))
+      end
+
+      visible
+    end
+  end
+end
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index ed86de819eb..c52d4d63382 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -2,7 +2,7 @@ module Ci
   class GitlabCiYamlProcessor
     class ValidationError < StandardError; end
 
-    include Gitlab::Ci::Config::Node::ValidationHelpers
+    include Gitlab::Ci::Config::Node::LegacyValidationHelpers
 
     DEFAULT_STAGES = %w(build test deploy)
     DEFAULT_STAGE = 'test'
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index b48d3592f16..adfd097736e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -4,8 +4,6 @@ module Gitlab
     # Base GitLab CI Configuration facade
     #
     class Config
-      delegate :valid?, :errors, to: :@global
-
       ##
       # Temporary delegations that should be removed after refactoring
       #
@@ -18,6 +16,14 @@ module Gitlab
         @global.process!
       end
 
+      def valid?
+        @global.valid?
+      end
+
+      def errors
+        @global.errors
+      end
+
       def to_hash
         @config
       end
diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb
index d60f87f3f94..374ff71d0f5 100644
--- a/lib/gitlab/ci/config/node/configurable.rb
+++ b/lib/gitlab/ci/config/node/configurable.rb
@@ -15,27 +15,24 @@ module Gitlab
         #
         module Configurable
           extend ActiveSupport::Concern
+          include Validatable
 
-          def allowed_nodes
-            self.class.allowed_nodes || {}
+          included do
+            validations do
+              validates :config, hash: true
+            end
           end
 
           private
 
-          def prevalidate!
-            unless @value.is_a?(Hash)
-              @errors << 'should be a configuration entry with hash value'
-            end
-          end
-
           def create_node(key, factory)
-            factory.with(value: @value[key])
-            factory.nullify! unless @value.has_key?(key)
+            factory.with(value: @config[key], key: key)
+            factory.nullify! unless @config.has_key?(key)
             factory.create!
           end
 
           class_methods do
-            def allowed_nodes
+            def nodes
               Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }]
             end
 
@@ -47,7 +44,6 @@ module Gitlab
 
               define_method(symbol) do
                 raise Entry::InvalidError unless valid?
-
                 @nodes[symbol].try(:value)
               end
 
diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb
index 52758a962f3..f044ef965e9 100644
--- a/lib/gitlab/ci/config/node/entry.rb
+++ b/lib/gitlab/ci/config/node/entry.rb
@@ -8,14 +8,14 @@ module Gitlab
         class Entry
           class InvalidError < StandardError; end
 
-          attr_accessor :description
+          attr_reader :config
+          attr_accessor :key, :description
 
-          def initialize(value)
-            @value = value
+          def initialize(config)
+            @config = config
             @nodes = {}
-            @errors = []
-
-            prevalidate!
+            @validator = self.class.validator.new(self)
+            @validator.validate
           end
 
           def process!
@@ -23,50 +23,54 @@ module Gitlab
             return unless valid?
 
             compose!
-
-            nodes.each(&:process!)
-            nodes.each(&:validate!)
+            process_nodes!
           end
 
           def nodes
             @nodes.values
           end
 
-          def valid?
-            errors.none?
-          end
-
           def leaf?
-            allowed_nodes.none?
+            self.class.nodes.none?
           end
 
-          def errors
-            @errors + nodes.map(&:errors).flatten
+          def key
+            @key || self.class.name.demodulize.underscore
           end
 
-          def allowed_nodes
-            {}
+          def valid?
+            errors.none?
           end
 
-          def validate!
-            raise NotImplementedError
+          def errors
+            @validator.full_errors +
+              nodes.map(&:errors).flatten
           end
 
           def value
             raise NotImplementedError
           end
 
-          private
+          def self.nodes
+            {}
+          end
 
-          def prevalidate!
+          def self.validator
+            Validator
           end
 
+          private
+
           def compose!
-            allowed_nodes.each do |key, essence|
+            self.class.nodes.each do |key, essence|
               @nodes[key] = create_node(key, essence)
             end
           end
 
+          def process_nodes!
+            nodes.each(&:process!)
+          end
+
           def create_node(key, essence)
             raise NotImplementedError
           end
diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb
index 787ca006f5a..025ae40ef94 100644
--- a/lib/gitlab/ci/config/node/factory.rb
+++ b/lib/gitlab/ci/config/node/factory.rb
@@ -30,6 +30,7 @@ module Gitlab
 
             @entry_class.new(@attributes[:value]).tap do |entry|
               entry.description = @attributes[:description]
+              entry.key = @attributes[:key]
             end
           end
         end
diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
index 72f648975dc..4d9a508796a 100644
--- a/lib/gitlab/ci/config/node/validation_helpers.rb
+++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Ci
     class Config
       module Node
-        module ValidationHelpers
+        module LegacyValidationHelpers
           private
 
           def validate_duration(value)
diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb
index 5072bf0db7d..c044f5c5e71 100644
--- a/lib/gitlab/ci/config/node/script.rb
+++ b/lib/gitlab/ci/config/node/script.rb
@@ -11,16 +11,14 @@ module Gitlab
         # implementation in Runner.
         #
         class Script < Entry
-          include ValidationHelpers
+          include Validatable
 
-          def value
-            @value.join("\n")
+          validations do
+            validates :config, array_of_strings: true
           end
 
-          def validate!
-            unless validate_array_of_strings(@value)
-              @errors << 'before_script should be an array of strings'
-            end
+          def value
+            @config.join("\n")
           end
         end
       end
diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/node/validatable.rb
new file mode 100644
index 00000000000..f6e2896dfb2
--- /dev/null
+++ b/lib/gitlab/ci/config/node/validatable.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module Validatable
+          extend ActiveSupport::Concern
+
+          class_methods do
+            def validator
+              validator = Class.new(Node::Validator)
+
+              if defined?(@validations)
+                @validations.each { |rules| validator.class_eval(&rules) }
+              end
+
+              validator
+            end
+
+            private
+
+            def validations(&block)
+              (@validations ||= []).append(block)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/node/validator.rb
new file mode 100644
index 00000000000..02edc9219c3
--- /dev/null
+++ b/lib/gitlab/ci/config/node/validator.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        class Validator < SimpleDelegator
+          include ActiveModel::Validations
+          include Node::Validators
+
+          def initialize(node)
+            super(node)
+            @node = node
+          end
+
+          def full_errors
+            errors.full_messages.map do |error|
+              "#{@node.key} #{error}".humanize
+            end
+          end
+
+          def self.name
+            'Validator'
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/node/validators.rb
new file mode 100644
index 00000000000..dc9cdb9a220
--- /dev/null
+++ b/lib/gitlab/ci/config/node/validators.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    class Config
+      module Node
+        module Validators
+          class ArrayOfStringsValidator < ActiveModel::EachValidator
+            include LegacyValidationHelpers
+
+            def validate_each(record, attribute, value)
+              unless validate_array_of_strings(value)
+                record.errors.add(attribute, 'should be an array of strings')
+              end
+            end
+          end
+
+          class HashValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless value.is_a?(Hash)
+                record.errors.add(attribute, 'should be a configuration entry hash')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 0e70d9282d5..82d1e1805c5 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -23,7 +23,11 @@ module Gitlab
       private
 
       def decompress_archive
-        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+        result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
+
+        raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
+
+        true
       end
     end
   end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index d209e04f7be..595b20a09bd 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -10,17 +10,22 @@ module Gitlab
       end
 
       def execute
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
-                                                  shared: @shared)
-        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
         end
+
+        remove_import_file
       end
 
       private
 
+      def import_file
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+      end
+
       def check_version!
         Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
       end
@@ -59,6 +64,10 @@ module Gitlab
       def wiki_repo_path
         File.join(@shared.export_path, 'project.wiki.bundle')
       end
+
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
     end
   end
 end