diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2016-01-14 15:57:33 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2016-01-14 15:57:33 +0300 |
commit | f03da18e3a925e88b46aabb5e095b90abe0f0131 (patch) | |
tree | 9cc013a388489cd6930e654428081a86dc62056a /lib | |
parent | f981da44ab88012db984e1457170067b345660c1 (diff) | |
parent | be764a3a20c7cecce2a047ddd46aff954c33b306 (diff) |
Merge branch 'ci/view-build-artifacts' into 'master'
Add browser for build artifacts
Discussion in #3426, closes #3426.
See merge request !2123
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers.rb | 4 | ||||
-rw-r--r-- | lib/ci/api/builds.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/ci/build/artifacts/metadata.rb | 109 | ||||
-rw-r--r-- | lib/gitlab/ci/build/artifacts/metadata/entry.rb | 119 |
4 files changed, 247 insertions, 7 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..d46b5c42967 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -289,12 +289,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..fb87637b94f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -78,11 +78,13 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - filename (generated by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -96,13 +98,20 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? + forbidden!('Build is not running!') unless build.running? - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) - if build.update_attributes(artifacts_file: file) - present build, with: Entities::Build + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + build.artifacts_file = artifacts + build.artifacts_metadata = metadata + + if build.save + present(build, with: Entities::Build) else render_validation_error!(build) end @@ -148,6 +157,7 @@ module Ci not_found! unless build authenticate_build_token!(build) build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..1344f5d120b --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,109 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + class ParserError < StandardError; end + + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + + attr_reader :file, :path, :full_version + + def initialize(file, path) + @file, @path = file, path + @full_version = read_version + end + + def version + @full_version.match(VERSION_PATTERN)[1] + end + + def errors + gzip do |gz| + read_string(gz) # version + errors = read_string(gz) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end + end + end + + def find_entries! + gzip do |gz| + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) + end + end + + def to_entry + entries = find_entries! + Entry.new(@path, entries) + end + + private + + def match_entries(gz) + entries = {} + match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} + + until gz.eof? do + begin + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + + next unless path.valid_encoding? && meta.valid_encoding? + next unless path =~ match_pattern + next if path =~ INVALID_PATH_PATTERN + + entries[path] = JSON.parse(meta, symbolize_names: true) + rescue JSON::ParserError, Encoding::CompatibilityError + next + end + end + + entries + end + + def read_version + gzip do |gz| + version_string = read_string(gz) + + unless version_string + raise ParserError, 'Artifacts metadata file empty!' + end + + unless version_string =~ VERSION_PATTERN + raise ParserError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_uint32(gz) + return nil unless string_size + gz.read(string_size) + end + + def gzip(&block) + Zlib::GzipReader.open(@file, &block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..25b71fc3275 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entries + attr_accessor :name + + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') + @entries = entries + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path) + end + + def empty? + children.empty? + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end + end + end + end +end |