diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2016-01-14 15:57:33 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2016-01-14 15:57:33 +0300 |
commit | f03da18e3a925e88b46aabb5e095b90abe0f0131 (patch) | |
tree | 9cc013a388489cd6930e654428081a86dc62056a | |
parent | f981da44ab88012db984e1457170067b345660c1 (diff) | |
parent | be764a3a20c7cecce2a047ddd46aff954c33b306 (diff) |
Merge branch 'ci/view-build-artifacts' into 'master'
Add browser for build artifacts
Discussion in #3426, closes #3426.
See merge request !2123
30 files changed, 1020 insertions, 87 deletions
diff --git a/CHANGELOG b/CHANGELOG index a0c56837f89..5739ea19564 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -50,6 +50,7 @@ v 8.4.0 (unreleased) v 8.3.4 - Use gitlab-workhorse 0.5.4 (fixes API routing bug) + - Add build artifacts browser v 8.3.3 - Preserve CE behavior with JIRA integration by only calling API if URL is set diff --git a/app/controllers/projects/artifacts_controller.rb b/app/controllers/projects/artifacts_controller.rb new file mode 100644 index 00000000000..dff0732bdfe --- /dev/null +++ b/app/controllers/projects/artifacts_controller.rb @@ -0,0 +1,56 @@ +class Projects::ArtifactsController < Projects::ApplicationController + layout 'project' + before_action :authorize_read_build_artifacts! + + def download + unless artifacts_file.file_storage? + return redirect_to artifacts_file.url + end + + unless artifacts_file.exists? + return not_found! + end + + send_file artifacts_file.path, disposition: 'attachment' + end + + def browse + return render_404 unless build.artifacts? + + directory = params[:path] ? "#{params[:path]}/" : '' + @entry = build.artifacts_metadata_entry(directory) + + return render_404 unless @entry.exists? + end + + def file + entry = build.artifacts_metadata_entry(params[:path]) + + if entry.exists? + render json: { archive: build.artifacts_file.path, + entry: Base64.encode64(entry.path) } + else + render json: {}, status: 404 + end + end + + private + + def build + @build ||= project.builds.unscoped.find_by!(id: params[:build_id]) + end + + def artifacts_file + @artifacts_file ||= build.artifacts_file + end + + def authorize_read_build_artifacts! + unless can?(current_user, :read_build_artifacts, @project) + if current_user.nil? + return authenticate_user! + else + return render_404 + end + end + end +end diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 39d3ba26ba2..0e965966ffa 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -2,7 +2,6 @@ class Projects::BuildsController < Projects::ApplicationController before_action :build, except: [:index, :cancel_all] before_action :authorize_manage_builds!, except: [:index, :show, :status] - before_action :authorize_download_build_artifacts!, only: [:download] layout "project" @@ -51,18 +50,6 @@ class Projects::BuildsController < Projects::ApplicationController redirect_to build_path(build) end - def download - unless artifacts_file.file_storage? - return redirect_to artifacts_file.url - end - - unless artifacts_file.exists? - return not_found! - end - - send_file artifacts_file.path, disposition: 'attachment' - end - def status render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha) end @@ -79,10 +66,6 @@ class Projects::BuildsController < Projects::ApplicationController @build ||= project.builds.unscoped.find_by!(id: params[:id]) end - def artifacts_file - build.artifacts_file - end - def build_path(build) namespace_project_build_path(build.project.namespace, build.project, build) end @@ -92,14 +75,4 @@ class Projects::BuildsController < Projects::ApplicationController return page_404 end end - - def authorize_download_build_artifacts! - unless can?(current_user, :download_build_artifacts, @project) - if current_user.nil? - return authenticate_user! - else - return render_404 - end - end - end end diff --git a/app/models/ability.rb b/app/models/ability.rb index 5a1a67db8e1..5375148a654 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -175,7 +175,7 @@ class Ability :create_merge_request, :create_wiki, :manage_builds, - :download_build_artifacts, + :read_build_artifacts, :push_code ] end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index a4779d06de8..6cc26abce66 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -30,10 +30,12 @@ # description :string(255) # artifacts_file :text # gl_project_id :integer +# artifacts_metadata :text # module Ci class Build < CommitStatus + include Gitlab::Application.routes.url_helpers LAZY_ATTRIBUTES = ['trace'] belongs_to :runner, class_name: 'Ci::Runner' @@ -49,6 +51,7 @@ module Ci scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) } mount_uploader :artifacts_file, ArtifactUploader + mount_uploader :artifacts_metadata, ArtifactUploader acts_as_taggable @@ -291,21 +294,18 @@ module Ci end def target_url - Gitlab::Application.routes.url_helpers. - namespace_project_build_url(project.namespace, project, self) + namespace_project_build_url(project.namespace, project, self) end def cancel_url if active? - Gitlab::Application.routes.url_helpers. - cancel_namespace_project_build_path(project.namespace, project, self) + cancel_namespace_project_build_path(project.namespace, project, self) end end def retry_url if retryable? - Gitlab::Application.routes.url_helpers. - retry_namespace_project_build_path(project.namespace, project, self) + retry_namespace_project_build_path(project.namespace, project, self) end end @@ -321,20 +321,35 @@ module Ci pending? && !any_runners_online? end - def download_url - if artifacts_file.exists? - Gitlab::Application.routes.url_helpers. - download_namespace_project_build_path(project.namespace, project, self) - end - end - def execute_hooks build_data = Gitlab::BuildDataBuilder.build(self) project.execute_hooks(build_data.dup, :build_hooks) project.execute_services(build_data.dup, :build_hooks) end + def artifacts? + artifacts_file.exists? + end + + def artifacts_download_url + if artifacts? + download_namespace_project_build_artifacts_path(project.namespace, project, self) + end + end + def artifacts_browse_url + if artifacts_browser_supported? + browse_namespace_project_build_artifacts_path(project.namespace, project, self) + end + end + + def artifacts_browser_supported? + artifacts? && artifacts_metadata.exists? + end + + def artifacts_metadata_entry(path) + Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path).to_entry + end private diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index ff479493474..4ce9b998dfc 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -131,7 +131,11 @@ class CommitStatus < ActiveRecord::Base false end - def download_url + def artifacts_download_url + nil + end + + def artifacts_browse_url nil end end diff --git a/app/views/admin/builds/_build.html.haml b/app/views/admin/builds/_build.html.haml index 6936e614346..c395bd908c3 100644 --- a/app/views/admin/builds/_build.html.haml +++ b/app/views/admin/builds/_build.html.haml @@ -60,8 +60,8 @@ %td .pull-right - - if current_user && can?(current_user, :download_build_artifacts, project) && build.download_url - = link_to build.download_url, title: 'Download artifacts' do + - if current_user && can?(current_user, :read_build_artifacts, project) && build.artifacts? + = link_to build.artifacts_download_url, title: 'Download artifacts' do %i.fa.fa-download - if current_user && can?(current_user, :manage_builds, build.project) - if build.active? diff --git a/app/views/projects/artifacts/_tree_directory.html.haml b/app/views/projects/artifacts/_tree_directory.html.haml new file mode 100644 index 00000000000..5b87d55efd5 --- /dev/null +++ b/app/views/projects/artifacts/_tree_directory.html.haml @@ -0,0 +1,7 @@ +%tr{ class: 'tree-item' } + %td.tree-item-file-name + = tree_icon('folder', '755', directory.name) + %span.str-truncated + = link_to directory.name, browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build, path: directory.path) + %td + %td diff --git a/app/views/projects/artifacts/_tree_file.html.haml b/app/views/projects/artifacts/_tree_file.html.haml new file mode 100644 index 00000000000..92c1648f726 --- /dev/null +++ b/app/views/projects/artifacts/_tree_file.html.haml @@ -0,0 +1,11 @@ +%tr{ class: 'tree-item' } + %td.tree-item-file-name + = tree_icon('file', '664', file.name) + %span.str-truncated + = file.name + %td + = number_to_human_size(file.metadata[:size], precision: 2) + %td + = link_to file_namespace_project_build_artifacts_path(@project.namespace, @project, @build, path: file.path), + class: 'btn btn-xs btn-default artifact-download' do + = icon('download') diff --git a/app/views/projects/artifacts/browse.html.haml b/app/views/projects/artifacts/browse.html.haml new file mode 100644 index 00000000000..1add7ef6bfb --- /dev/null +++ b/app/views/projects/artifacts/browse.html.haml @@ -0,0 +1,24 @@ +- page_title 'Artifacts', "#{@build.name} (##{@build.id})", 'Builds' += render 'projects/builds/header_title' + +#tree-holder.tree-holder + .gray-content-block.top-block.clearfix + .pull-right + = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), + class: 'btn btn-default' do + = icon('download') + Download artifacts archive + +%div.tree-content-holder + .table-holder + %table.table.tree-table.table-striped + %thead + %tr + %th Name + %th Size + %th Download + = render partial: 'tree_directory', collection: @entry.directories(parent: true), as: :directory + = render partial: 'tree_file', collection: @entry.files, as: :file + +- if @entry.empty? + .center Empty diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml index 5b7ecce86ab..f393d818656 100644 --- a/app/views/projects/builds/show.html.haml +++ b/app/views/projects/builds/show.html.haml @@ -89,9 +89,15 @@ Test coverage %h1 #{@build.coverage}% - - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url - .build-widget.center - = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary' + - if current_user && can?(current_user, :read_build_artifacts, @project) && @build.artifacts? + + .build-widget.artifacts + %h4.title Build artifacts + .center + .btn-group{ role: :group } + = link_to "Download", @build.artifacts_download_url, class: 'btn btn-sm btn-primary' + - if @build.artifacts_browser_supported? + = link_to "Browse", @build.artifacts_browse_url, class: 'btn btn-sm btn-primary' .build-widget %h4.title diff --git a/app/views/projects/commit_statuses/_commit_status.html.haml b/app/views/projects/commit_statuses/_commit_status.html.haml index 74a05df24d3..1736dccaf3c 100644 --- a/app/views/projects/commit_statuses/_commit_status.html.haml +++ b/app/views/projects/commit_statuses/_commit_status.html.haml @@ -66,8 +66,8 @@ %td .pull-right - - if current_user && can?(current_user, :download_build_artifacts, commit_status.project) && commit_status.download_url - = link_to commit_status.download_url, title: 'Download artifacts' do + - if current_user && can?(current_user, :read_build_artifacts, commit_status.project) && commit_status.artifacts? + = link_to commit_status.artifacts_download_url, title: 'Download artifacts' do %i.fa.fa-download - if current_user && can?(current_user, :manage_builds, commit_status.project) - if commit_status.active? diff --git a/config/routes.rb b/config/routes.rb index 05d6ff1e884..0a29782f55b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -604,9 +604,14 @@ Rails.application.routes.draw do member do get :status post :cancel - get :download post :retry end + + resource :artifacts, only: [] do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + end end resources :hooks, only: [:index, :create, :destroy], constraints: { id: /\d+/ } do diff --git a/db/fixtures/development/14_builds.rb b/db/fixtures/development/14_builds.rb new file mode 100644 index 00000000000..03a12323845 --- /dev/null +++ b/db/fixtures/development/14_builds.rb @@ -0,0 +1,79 @@ +class Gitlab::Seeder::Builds + BUILD_STATUSES = %w(running pending success failed canceled) + + def initialize(project) + @project = project + end + + def seed! + ci_commits.each do |ci_commit| + build = Ci::Build.new(build_attributes_for(ci_commit)) + + artifacts_cache_file(artifacts_archive_path) do |file| + build.artifacts_file = file + end + + artifacts_cache_file(artifacts_metadata_path) do |file| + build.artifacts_metadata = file + end + + begin + build.save! + print '.' + rescue ActiveRecord::RecordInvalid + print 'F' + end + end + end + + def ci_commits + commits = @project.repository.commits('master', nil, 5) + commits_sha = commits.map { |commit| commit.raw.id } + commits_sha.map do |sha| + @project.ensure_ci_commit(sha) + end + rescue + [] + end + + def build_attributes_for(ci_commit) + { name: 'test build', commands: "$ build command", + stage: 'test', stage_idx: 1, ref: 'master', + user_id: build_user, gl_project_id: @project.id, + status: build_status, commit_id: ci_commit.id, + created_at: Time.now, updated_at: Time.now } + end + + def build_user + @project.team.users.sample + end + + def build_status + BUILD_STATUSES.sample + end + + def artifacts_archive_path + Rails.root + 'spec/fixtures/ci_build_artifacts.zip' + end + + def artifacts_metadata_path + Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz' + + end + + def artifacts_cache_file(file_path) + cache_path = file_path.to_s.gsub('ci_', "p#{@project.id}_") + + FileUtils.copy(file_path, cache_path) + File.open(cache_path) do |file| + yield file + end + end +end + +Gitlab::Seeder.quiet do + Project.all.sample(10).each do |project| + project_builds = Gitlab::Seeder::Builds.new(project) + project_builds.seed! + end +end diff --git a/db/migrate/20151230132518_add_artifacts_metadata_to_ci_build.rb b/db/migrate/20151230132518_add_artifacts_metadata_to_ci_build.rb new file mode 100644 index 00000000000..6c282fc5039 --- /dev/null +++ b/db/migrate/20151230132518_add_artifacts_metadata_to_ci_build.rb @@ -0,0 +1,5 @@ +class AddArtifactsMetadataToCiBuild < ActiveRecord::Migration + def change + add_column :ci_builds, :artifacts_metadata, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 42c3e79f9d7..2fc8c4d5ed4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -123,6 +123,7 @@ ActiveRecord::Schema.define(version: 20160113111034) do t.string "description" t.text "artifacts_file" t.integer "gl_project_id" + t.text "artifacts_metadata" end add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree diff --git a/features/project/builds.feature b/features/project/builds.feature new file mode 100644 index 00000000000..c00b0a7ae07 --- /dev/null +++ b/features/project/builds.feature @@ -0,0 +1,58 @@ +Feature: Project Builds + Background: + Given I sign in as a user + And I own a project + And CI is enabled + And I have recent build for my project + + Scenario: I browse build summary page + When I visit recent build summary page + Then I see summary for build + And I see build trace + + Scenario: I download build artifacts + Given recent build has artifacts available + When I visit recent build summary page + And I click artifacts download button + Then download of build artifacts archive starts + + Scenario: I browse build artifacts + Given recent build has artifacts available + And recent build has artifacts metadata available + When I visit recent build summary page + And I click artifacts browse button + Then I should see content of artifacts archive + + Scenario: I browse subdirectory of build artifacts + Given recent build has artifacts available + And recent build has artifacts metadata available + When I visit recent build summary page + And I click artifacts browse button + And I click link to subdirectory within build artifacts + Then I should see content of subdirectory within artifacts archive + + Scenario: I browse directory with UTF-8 characters in name + Given recent build has artifacts available + And recent build has artifacts metadata available + And recent build artifacts contain directory with UTF-8 characters + When I visit recent build summary page + And I click artifacts browse button + And I navigate to directory with UTF-8 characters in name + Then I should see content of directory with UTF-8 characters in name + + Scenario: I try to browse directory with invalid UTF-8 characters in name + Given recent build has artifacts available + And recent build has artifacts metadata available + And recent build artifacts contain directory with invalid UTF-8 characters + When I visit recent build summary page + And I click artifacts browse button + And I navigate to parent directory of directory with invalid name + Then I should not see directory with invalid name on the list + + Scenario: I download a single file from build artifacts + Given recent build has artifacts available + And recent build has artifacts metadata available + When I visit recent build summary page + And I click artifacts browse button + And I click download button for a file within build artifacts + Then download of a file extracted from build artifacts should start diff --git a/features/steps/project/builds.rb b/features/steps/project/builds.rb new file mode 100644 index 00000000000..28395281077 --- /dev/null +++ b/features/steps/project/builds.rb @@ -0,0 +1,89 @@ +class Spinach::Features::ProjectBuilds < Spinach::FeatureSteps + include SharedAuthentication + include SharedProject + include SharedBuilds + include RepoHelpers + + step 'I see summary for build' do + expect(page).to have_content "Build ##{@build.id}" + end + + step 'I see build trace' do + expect(page).to have_css '#build-trace' + end + + step 'I click artifacts download button' do + page.within('.artifacts') { click_link 'Download' } + end + + step 'download of build artifacts archive starts' do + expect(page.response_headers['Content-Type']).to eq 'application/zip' + expect(page.response_headers['Content-Transfer-Encoding']).to eq 'binary' + end + + step 'I click artifacts browse button' do + page.within('.artifacts') { click_link 'Browse' } + end + + step 'I should see content of artifacts archive' do + page.within('.tree-table') do + expect(page).to have_no_content '..' + expect(page).to have_content 'other_artifacts_0.1.2' + expect(page).to have_content 'ci_artifacts.txt' + expect(page).to have_content 'rails_sample.jpg' + end + end + + step 'I click link to subdirectory within build artifacts' do + page.within('.tree-table') { click_link 'other_artifacts_0.1.2' } + end + + step 'I should see content of subdirectory within artifacts archive' do + page.within('.tree-table') do + expect(page).to have_content '..' + expect(page).to have_content 'another-subdirectory' + expect(page).to have_content 'doc_sample.txt' + end + end + + step 'recent build artifacts contain directory with UTF-8 characters' do + # metadata fixture contains relevant directory + end + + step 'I navigate to directory with UTF-8 characters in name' do + page.within('.tree-table') { click_link 'tests_encoding' } + page.within('.tree-table') { click_link 'utf8 test dir ✓' } + end + + step 'I should see content of directory with UTF-8 characters in name' do + page.within('.tree-table') do + expect(page).to have_content '..' + expect(page).to have_content 'regular_file_2' + end + end + + step 'recent build artifacts contain directory with invalid UTF-8 characters' do + # metadata fixture contains relevant directory + end + + step 'I navigate to parent directory of directory with invalid name' do + page.within('.tree-table') { click_link 'tests_encoding' } + end + + step 'I should not see directory with invalid name on the list' do + page.within('.tree-table') do + expect(page).to have_no_content('non-utf8-dir') + end + end + + step 'I click download button for a file within build artifacts' do + page.within('.tree-table') { first('.artifact-download').click } + end + + step 'download of a file extracted from build artifacts should start' do + # this will be accelerated by Workhorse + response_json = JSON.parse(page.body, symbolize_names: true) + expect(response_json[:archive]).to end_with('build_artifacts.zip') + expect(response_json[:entry]).to eq Base64.encode64('ci_artifacts.txt') + end +end diff --git a/features/steps/shared/builds.rb b/features/steps/shared/builds.rb new file mode 100644 index 00000000000..a83d74e5946 --- /dev/null +++ b/features/steps/shared/builds.rb @@ -0,0 +1,28 @@ +module SharedBuilds + include Spinach::DSL + + step 'CI is enabled' do + @project.enable_ci + end + + step 'I have recent build for my project' do + ci_commit = create :ci_commit, project: @project, sha: sample_commit.id + @build = create :ci_build, commit: ci_commit + end + + step 'I visit recent build summary page' do + visit namespace_project_build_path(@project.namespace, @project, @build) + end + + step 'recent build has artifacts available' do + artifacts = Rails.root + 'spec/fixtures/ci_build_artifacts.zip' + archive = fixture_file_upload(artifacts, 'application/zip') + @build.update_attributes(artifacts_file: archive) + end + + step 'recent build has artifacts metadata available' do + metadata = Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz' + gzip = fixture_file_upload(metadata, 'application/x-gzip') + @build.update_attributes(artifacts_metadata: gzip) + end +end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..d46b5c42967 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -289,12 +289,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..fb87637b94f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -78,11 +78,13 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - filename (generated by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -96,13 +98,20 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? + forbidden!('Build is not running!') unless build.running? - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) - if build.update_attributes(artifacts_file: file) - present build, with: Entities::Build + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + build.artifacts_file = artifacts + build.artifacts_metadata = metadata + + if build.save + present(build, with: Entities::Build) else render_validation_error!(build) end @@ -148,6 +157,7 @@ module Ci not_found! unless build authenticate_build_token!(build) build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..1344f5d120b --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,109 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + class ParserError < StandardError; end + + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + + attr_reader :file, :path, :full_version + + def initialize(file, path) + @file, @path = file, path + @full_version = read_version + end + + def version + @full_version.match(VERSION_PATTERN)[1] + end + + def errors + gzip do |gz| + read_string(gz) # version + errors = read_string(gz) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end + end + end + + def find_entries! + gzip do |gz| + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) + end + end + + def to_entry + entries = find_entries! + Entry.new(@path, entries) + end + + private + + def match_entries(gz) + entries = {} + match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} + + until gz.eof? do + begin + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + + next unless path.valid_encoding? && meta.valid_encoding? + next unless path =~ match_pattern + next if path =~ INVALID_PATH_PATTERN + + entries[path] = JSON.parse(meta, symbolize_names: true) + rescue JSON::ParserError, Encoding::CompatibilityError + next + end + end + + entries + end + + def read_version + gzip do |gz| + version_string = read_string(gz) + + unless version_string + raise ParserError, 'Artifacts metadata file empty!' + end + + unless version_string =~ VERSION_PATTERN + raise ParserError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_uint32(gz) + return nil unless string_size + gz.read(string_size) + end + + def gzip(&block) + Zlib::GzipReader.open(@file, &block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..25b71fc3275 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entries + attr_accessor :name + + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') + @entries = entries + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path) + end + + def empty? + children.empty? + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end + end + end + end +end diff --git a/spec/features/builds_spec.rb b/spec/features/builds_spec.rb index 240e56839df..d37bd103714 100644 --- a/spec/features/builds_spec.rb +++ b/spec/features/builds_spec.rb @@ -80,7 +80,11 @@ describe "Builds" do visit namespace_project_build_path(@project.namespace, @project, @build) end - it { expect(page).to have_content 'Download artifacts' } + it 'has button to download artifacts' do + page.within('.artifacts') do + expect(page).to have_content 'Download' + end + end end end @@ -111,7 +115,7 @@ describe "Builds" do before do @build.update_attributes(artifacts_file: artifacts_file) visit namespace_project_build_path(@project.namespace, @project, @build) - click_link 'Download artifacts' + page.within('.artifacts') { click_link 'Download' } end it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) } diff --git a/spec/fixtures/ci_build_artifacts.zip b/spec/fixtures/ci_build_artifacts.zip Binary files differnew file mode 100644 index 00000000000..dae976d918e --- /dev/null +++ b/spec/fixtures/ci_build_artifacts.zip diff --git a/spec/fixtures/ci_build_artifacts_metadata.gz b/spec/fixtures/ci_build_artifacts_metadata.gz Binary files differnew file mode 100644 index 00000000000..fe9d4c8c661 --- /dev/null +++ b/spec/fixtures/ci_build_artifacts_metadata.gz diff --git a/spec/lib/gitlab/ci/build/artifacts/metadata/entry_spec.rb b/spec/lib/gitlab/ci/build/artifacts/metadata/entry_spec.rb new file mode 100644 index 00000000000..41257103ead --- /dev/null +++ b/spec/lib/gitlab/ci/build/artifacts/metadata/entry_spec.rb @@ -0,0 +1,168 @@ +require 'spec_helper' + +describe Gitlab::Ci::Build::Artifacts::Metadata::Entry do + let(:entries) do + { 'path/' => {}, + 'path/dir_1/' => {}, + 'path/dir_1/file_1' => {}, + 'path/dir_1/file_b' => {}, + 'path/dir_1/subdir/' => {}, + 'path/dir_1/subdir/subfile' => {}, + 'path/second_dir' => {}, + 'path/second_dir/dir_3/file_2' => {}, + 'path/second_dir/dir_3/file_3'=> {}, + 'another_directory/'=> {}, + 'another_file' => {}, + '/file/with/absolute_path' => {} } + end + + def path(example) + entry(example.metadata[:path]) + end + + def entry(path) + described_class.new(path, entries) + end + + describe '/file/with/absolute_path', path: '/file/with/absolute_path' do + subject { |example| path(example) } + + it { is_expected.to be_file } + it { is_expected.to have_parent } + + describe '#basename' do + subject { |example| path(example).basename } + it { is_expected.to eq 'absolute_path' } + end + end + + describe 'path/dir_1/', path: 'path/dir_1/' do + subject { |example| path(example) } + it { is_expected.to have_parent } + it { is_expected.to be_directory } + + describe '#basename' do + subject { |example| path(example).basename } + it { is_expected.to eq 'dir_1/' } + end + + describe '#name' do + subject { |example| path(example).name } + it { is_expected.to eq 'dir_1' } + end + + describe '#parent' do + subject { |example| path(example).parent } + it { is_expected.to eq entry('path/') } + end + + describe '#children' do + subject { |example| path(example).children } + + it { is_expected.to all(be_an_instance_of described_class) } + it do + is_expected.to contain_exactly entry('path/dir_1/file_1'), + entry('path/dir_1/file_b'), + entry('path/dir_1/subdir/') + end + end + + describe '#files' do + subject { |example| path(example).files } + + it { is_expected.to all(be_file) } + it { is_expected.to all(be_an_instance_of described_class) } + it do + is_expected.to contain_exactly entry('path/dir_1/file_1'), + entry('path/dir_1/file_b') + end + end + + describe '#directories' do + context 'without options' do + subject { |example| path(example).directories } + + it { is_expected.to all(be_directory) } + it { is_expected.to all(be_an_instance_of described_class) } + it { is_expected.to contain_exactly entry('path/dir_1/subdir/') } + end + + context 'with option parent: true' do + subject { |example| path(example).directories(parent: true) } + + it { is_expected.to all(be_directory) } + it { is_expected.to all(be_an_instance_of described_class) } + it do + is_expected.to contain_exactly entry('path/dir_1/subdir/'), + entry('path/') + end + end + + describe '#nodes' do + subject { |example| path(example).nodes } + it { is_expected.to eq 2 } + end + + describe '#exists?' do + subject { |example| path(example).exists? } + it { is_expected.to be true } + end + + describe '#empty?' do + subject { |example| path(example).empty? } + it { is_expected.to be false } + end + end + end + + describe 'empty path', path: '' do + subject { |example| path(example) } + it { is_expected.to_not have_parent } + + describe '#children' do + subject { |example| path(example).children } + it { expect(subject.count).to eq 3 } + end + + end + + describe 'path/dir_1/subdir/subfile', path: 'path/dir_1/subdir/subfile' do + describe '#nodes' do + subject { |example| path(example).nodes } + it { is_expected.to eq 4 } + end + end + + describe 'non-existent/', path: 'non-existent/' do + describe '#empty?' do + subject { |example| path(example).empty? } + it { is_expected.to be true } + end + + describe '#exists?' do + subject { |example| path(example).exists? } + it { is_expected.to be false } + end + end + + describe 'another_directory/', path: 'another_directory/' do + describe '#empty?' do + subject { |example| path(example).empty? } + it { is_expected.to be true } + end + end + + describe '#metadata' do + let(:entries) do + { 'path/' => { name: '/path/' }, + 'path/file1' => { name: '/path/file1' }, + 'path/file2' => { name: '/path/file2' } } + end + + subject do + described_class.new('path/file1', entries).metadata[:name] + end + + it { is_expected.to eq '/path/file1' } + end +end diff --git a/spec/lib/gitlab/ci/build/artifacts/metadata_spec.rb b/spec/lib/gitlab/ci/build/artifacts/metadata_spec.rb new file mode 100644 index 00000000000..828eedfa7b0 --- /dev/null +++ b/spec/lib/gitlab/ci/build/artifacts/metadata_spec.rb @@ -0,0 +1,84 @@ +require 'spec_helper' + +describe Gitlab::Ci::Build::Artifacts::Metadata do + def metadata(path = '') + described_class.new(metadata_file_path, path) + end + + let(:metadata_file_path) do + Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz' + end + + context 'metadata file exists' do + describe '#find_entries! empty string' do + subject { metadata('').find_entries! } + + it 'matches correct paths' do + expect(subject.keys).to contain_exactly 'ci_artifacts.txt', + 'other_artifacts_0.1.2/', + 'rails_sample.jpg', + 'tests_encoding/' + end + + it 'matches metadata for every path' do + expect(subject.keys.count).to eq 4 + end + + it 'return Hashes for each metadata' do + expect(subject.values).to all(be_kind_of(Hash)) + end + end + + describe '#find_entries! other_artifacts_0.1.2/' do + subject { metadata('other_artifacts_0.1.2/').find_entries! } + + it 'matches correct paths' do + expect(subject.keys). + to contain_exactly 'other_artifacts_0.1.2/', + 'other_artifacts_0.1.2/doc_sample.txt', + 'other_artifacts_0.1.2/another-subdirectory/' + end + end + + describe '#find_entries! other_artifacts_0.1.2/another-subdirectory/' do + subject { metadata('other_artifacts_0.1.2/another-subdirectory/').find_entries! } + + it 'matches correct paths' do + expect(subject.keys). + to contain_exactly 'other_artifacts_0.1.2/another-subdirectory/', + 'other_artifacts_0.1.2/another-subdirectory/empty_directory/', + 'other_artifacts_0.1.2/another-subdirectory/banana_sample.gif' + end + end + + describe '#to_entry' do + subject { metadata('').to_entry } + it { is_expected.to be_an_instance_of(Gitlab::Ci::Build::Artifacts::Metadata::Entry) } + end + + describe '#full_version' do + subject { metadata('').full_version } + it { is_expected.to eq 'GitLab Build Artifacts Metadata 0.0.1' } + end + + describe '#version' do + subject { metadata('').version } + it { is_expected.to eq '0.0.1' } + end + + describe '#errors' do + subject { metadata('').errors } + it { is_expected.to eq({}) } + end + end + + context 'metadata file does not exist' do + let(:metadata_file_path) { '' } + + describe '#find_entries!' do + it 'raises error' do + expect { metadata.find_entries! }.to raise_error(Errno::ENOENT) + end + end + end +end diff --git a/spec/models/build_spec.rb b/spec/models/build_spec.rb index 1c22e3cb7c4..0e13456723d 100644 --- a/spec/models/build_spec.rb +++ b/spec/models/build_spec.rb @@ -1,28 +1,3 @@ -# == Schema Information -# -# Table name: builds -# -# id :integer not null, primary key -# project_id :integer -# status :string(255) -# finished_at :datetime -# trace :text -# created_at :datetime -# updated_at :datetime -# started_at :datetime -# runner_id :integer -# commit_id :integer -# coverage :float -# commands :text -# job_id :integer -# name :string(255) -# deploy :boolean default(FALSE) -# options :text -# allow_failure :boolean default(FALSE), not null -# stage :string(255) -# trigger_request_id :integer -# - require 'spec_helper' describe Ci::Build, models: true do @@ -368,21 +343,75 @@ describe Ci::Build, models: true do end end - describe :download_url do - subject { build.download_url } + describe :artifacts_download_url do + subject { build.artifacts_download_url } it "should be nil if artifact doesn't exist" do build.update_attributes(artifacts_file: nil) is_expected.to be_nil end - it 'should be nil if artifact exist' do + it 'should not be nil if artifact exist' do gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') build.update_attributes(artifacts_file: gif) is_expected.to_not be_nil end end + describe :artifacts_browse_url do + subject { build.artifacts_browse_url } + + it "should be nil if artifacts browser is unsupported" do + allow(build).to receive(:artifacts_browser_supported?).and_return(false) + is_expected.to be_nil + end + + it 'should not be nil if artifacts browser is supported' do + allow(build).to receive(:artifacts_browser_supported?).and_return(true) + is_expected.to_not be_nil + end + end + + describe :artifacts? do + subject { build.artifacts? } + + context 'artifacts archive does not exist' do + before { build.update_attributes(artifacts_file: nil) } + it { is_expected.to be_falsy } + end + + context 'artifacts archive exists' do + before do + gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') + build.update_attributes(artifacts_file: gif) + end + + it { is_expected.to be_truthy } + end + end + + + describe :artifacts_browser_supported? do + subject { build.artifacts_browser_supported? } + context 'artifacts metadata does not exist' do + it { is_expected.to be_falsy } + end + + context 'artifacts archive is a zip file and metadata exists' do + before do + fixture_dir = Rails.root + 'spec/fixtures/' + archive = fixture_file_upload(fixture_dir + 'ci_build_artifacts.zip', + 'application/zip') + metadata = fixture_file_upload(fixture_dir + 'ci_build_artifacts_metadata.gz', + 'application/x-gzip') + build.update_attributes(artifacts_file: archive) + build.update_attributes(artifacts_metadata: metadata) + end + + it { is_expected.to be_truthy } + end + end + describe :repo_url do let(:build) { FactoryGirl.create :ci_build } let(:project) { build.project } diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index c27e87c4acc..648ea0d5f50 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -210,6 +210,52 @@ describe Ci::API::API do end end + context 'should post artifacts file and metadata file' do + let!(:artifacts) { file_upload } + let!(:metadata) { file_upload2 } + + let(:stored_artifacts_file) { build.reload.artifacts_file.file } + let(:stored_metadata_file) { build.reload.artifacts_metadata.file } + + before do + build.run! + post(post_url, post_data, headers_with_token) + end + + context 'post data accelerated by workhorse is correct' do + let(:post_data) do + { 'file.path' => artifacts.path, + 'file.name' => artifacts.original_filename, + 'metadata.path' => metadata.path, + 'metadata.name' => metadata.original_filename } + end + + it 'responds with valid status' do + expect(response.status).to eq(201) + end + + it 'stores artifacts and artifacts metadata' do + expect(stored_artifacts_file.original_filename).to eq(artifacts.original_filename) + expect(stored_metadata_file.original_filename).to eq(metadata.original_filename) + end + end + + context 'no artifacts file in post data' do + let(:post_data) do + { 'metadata' => metadata } + end + + it 'is expected to respond with bad request' do + expect(response.status).to eq(400) + end + + it 'does not store metadata' do + expect(stored_metadata_file).to be_nil + end + end + end + + context "should fail to post too large artifact" do before do build.run! |