Merge branch 'ci/view-build-artifacts' into 'master'

Add browser for build artifacts

Discussion in #3426, closes #3426.

See merge request !2123

4 files changed, 247 insertions, 7 deletions

diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index a4df810e755..d46b5c42967 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -289,12 +289,14 @@ module API
 
     # file helpers
 
-    def uploaded_file!(field, uploads_path)
+    def uploaded_file(field, uploads_path)
       if params[field]
         bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
         return params[field]
       end
 
+      return nil unless params["#{field}.path"] && params["#{field}.name"]
+
       # sanitize file paths
       # this requires all paths to exist
       required_attributes! %W(#{field}.path)
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 15faa6edd84..fb87637b94f 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -78,11 +78,13 @@ module Ci
         # Parameters:
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
-        #   file (required) - The uploaded file
+        #   file (required) - Artifacts file
         # Parameters (accelerated by GitLab Workhorse):
         #   file.path - path to locally stored body (generated by Workhorse)
         #   file.name - real filename as send in Content-Disposition
         #   file.type - real content type as send in Content-Type
+        #   metadata.path - path to locally stored body (generated by Workhorse)
+        #   metadata.name - filename (generated by Workhorse)
         # Headers:
         #   BUILD-TOKEN (required) - The build authorization token, the same as token
         # Body:
@@ -96,13 +98,20 @@ module Ci
           build = Ci::Build.find_by_id(params[:id])
           not_found! unless build
           authenticate_build_token!(build)
-          forbidden!('build is not running') unless build.running?
+          forbidden!('Build is not running!') unless build.running?
 
-          file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path)
-          file_to_large! unless file.size < max_artifacts_size
+          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+          artifacts = uploaded_file(:file, artifacts_upload_path)
+          metadata = uploaded_file(:metadata, artifacts_upload_path)
 
-          if build.update_attributes(artifacts_file: file)
-            present build, with: Entities::Build
+          bad_request!('Missing artifacts file!') unless artifacts
+          file_to_large! unless artifacts.size < max_artifacts_size
+
+          build.artifacts_file = artifacts
+          build.artifacts_metadata = metadata
+
+          if build.save
+            present(build, with: Entities::Build)
           else
             render_validation_error!(build)
           end
@@ -148,6 +157,7 @@ module Ci
           not_found! unless build
           authenticate_build_token!(build)
           build.remove_artifacts_file!
+          build.remove_artifacts_metadata!
         end
       end
     end
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
new file mode 100644
index 00000000000..1344f5d120b
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -0,0 +1,109 @@
+require 'zlib'
+require 'json'
+
+module Gitlab
+  module Ci
+    module Build
+      module Artifacts
+        class Metadata
+          class ParserError < StandardError; end
+
+          VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
+          INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
+
+          attr_reader :file, :path, :full_version
+
+          def initialize(file, path)
+            @file, @path = file, path
+            @full_version = read_version
+          end
+
+          def version
+            @full_version.match(VERSION_PATTERN)[1]
+          end
+
+          def errors
+            gzip do |gz|
+              read_string(gz) # version
+              errors = read_string(gz)
+              raise ParserError, 'Errors field not found!' unless errors
+
+              begin
+                JSON.parse(errors)
+              rescue JSON::ParserError
+                raise ParserError, 'Invalid errors field!'
+              end
+            end
+          end
+
+          def find_entries!
+            gzip do |gz|
+              2.times { read_string(gz) } # version and errors fields
+              match_entries(gz)
+            end
+          end
+
+          def to_entry
+            entries = find_entries!
+            Entry.new(@path, entries)
+          end
+
+          private
+
+          def match_entries(gz)
+            entries = {}
+            match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$}
+
+            until gz.eof? do
+              begin
+                path = read_string(gz).force_encoding('UTF-8')
+                meta = read_string(gz).force_encoding('UTF-8')
+               
+                next unless path.valid_encoding? && meta.valid_encoding?
+                next unless path =~ match_pattern
+                next if path =~ INVALID_PATH_PATTERN
+
+                entries[path] = JSON.parse(meta, symbolize_names: true)
+              rescue JSON::ParserError, Encoding::CompatibilityError
+                next
+              end
+            end
+
+            entries
+          end
+
+          def read_version
+            gzip do |gz|
+              version_string = read_string(gz)
+
+              unless version_string
+                raise ParserError, 'Artifacts metadata file empty!'
+              end
+
+              unless version_string =~ VERSION_PATTERN
+                raise ParserError, 'Invalid version!'
+              end
+
+              version_string.chomp
+            end
+          end
+
+          def read_uint32(gz)
+            binary = gz.read(4)
+            binary.unpack('L>')[0] if binary
+          end
+
+          def read_string(gz)
+            string_size = read_uint32(gz)
+            return nil unless string_size
+            gz.read(string_size)
+          end
+
+          def gzip(&block)
+            Zlib::GzipReader.open(@file, &block)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
new file mode 100644
index 00000000000..25b71fc3275
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -0,0 +1,119 @@
+module Gitlab
+  module Ci::Build::Artifacts
+    class Metadata
+      ##
+      # Class that represents an entry (path and metadata) to a file or
+      # directory in GitLab CI Build Artifacts binary file / archive
+      #
+      # This is IO-operations safe class, that does similar job to
+      # Ruby's Pathname but without the risk of accessing filesystem.
+      #
+      # This class is working only with UTF-8 encoded paths.
+      #
+      class Entry
+        attr_reader :path, :entries
+        attr_accessor :name
+
+        def initialize(path, entries)
+          @path = path.dup.force_encoding('UTF-8')
+          @entries = entries
+
+          if path.include?("\0")
+            raise ArgumentError, 'Path contains zero byte character!'
+          end
+
+          unless path.valid_encoding?
+            raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
+          end
+        end
+
+        def directory?
+          blank_node? || @path.end_with?('/')
+        end
+
+        def file?
+          !directory?
+        end
+
+        def has_parent?
+          nodes > 0
+        end
+
+        def parent
+          return nil unless has_parent?
+          self.class.new(@path.chomp(basename), @entries)
+        end
+
+        def basename
+          (directory? && !blank_node?) ? name + '/' : name
+        end
+
+        def name
+          @name || @path.split('/').last.to_s
+        end
+
+        def children
+          return [] unless directory?
+          return @children if @children
+
+          child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
+          @children = select_entries { |path| path =~ child_pattern }
+        end
+
+        def directories(opts = {})
+          return [] unless directory?
+          dirs = children.select(&:directory?)
+          return dirs unless has_parent? && opts[:parent]
+
+          dotted_parent = parent
+          dotted_parent.name = '..'
+          dirs.prepend(dotted_parent)
+        end
+
+        def files
+          return [] unless directory?
+          children.select(&:file?)
+        end
+
+        def metadata
+          @entries[@path] || {}
+        end
+
+        def nodes
+          @path.count('/') + (file? ? 1 : 0)
+        end
+
+        def blank_node?
+          @path.empty? # "" is considered to be './'
+        end
+
+        def exists?
+          blank_node? || @entries.include?(@path)
+        end
+
+        def empty?
+          children.empty?
+        end
+
+        def to_s
+          @path
+        end
+
+        def ==(other)
+          @path == other.path && @entries == other.entries
+        end
+
+        def inspect
+          "#{self.class.name}: #{@path}"
+        end
+
+        private
+
+        def select_entries
+          selected = @entries.select { |path, _metadata| yield path }
+          selected.map { |path, _metadata| self.class.new(path, @entries) }
+        end
+      end
+    end
+  end
+end