diff options
author | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-01-14 23:58:17 +0300 |
---|---|---|
committer | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-01-14 23:58:17 +0300 |
commit | 768721dc5895467130b9a3ada9c740a050f4fdff (patch) | |
tree | d96448edb9490825e8384a1b42bbfafa98140a33 /lib | |
parent | 7ed5bc5c9b69e6caf45541dc8f820aa59a20c160 (diff) | |
parent | 2f2aada3b6b900fdff276aafc3f93e5a1c1465c6 (diff) |
Merge branch 'master' into ci/api-triggers
* master: (150 commits)
fixes margin and padding
Update mailroom/postfix documentation [ci skip]
Fix css mess around git clone panel. Align it properly
Fix missing padding for user/group pages
Fix parse_gollum_tags matcher
Update documentation on Banzai::Filter::GollumTagsFilter
Add tests for the wiki pipeline
Refactoring Banzai::Filter::GollumTagsFilter
Make sure the .git is at the end on Gitlab::GithubImport::WikiFormatter
Remove GollumTagsPipeline
Refactoring Gitlab::GithubImport::Importer
Remove unnecessary brackets on WIKI_SLUG_ID route constraints
Move js function to removing accents to vendor/assets/javascripts
Update CHANGELOG
Use the WikiPipeline when rendering the wiki markdown content
Add Banzai::Filter::GollumTagsFilter for parsing Gollum's tags in HTML
Relax constraints for wiki slug
Import GitHub wiki into GitLab
Move Ci::Build#available_statuses to AVAILABLE_STATUSES constant in CommitStatus
Revert changes to how the notes are paginated in the API
...
Conflicts:
doc/api/README.md
lib/api/entities.rb
Diffstat (limited to 'lib')
55 files changed, 652 insertions, 94 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 098dd975840..7efe0a0262f 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -54,6 +54,7 @@ module API mount Keys mount Tags mount Triggers + mount Builds mount Variables end end diff --git a/lib/api/builds.rb b/lib/api/builds.rb new file mode 100644 index 00000000000..d293f988165 --- /dev/null +++ b/lib/api/builds.rb @@ -0,0 +1,149 @@ +module API + # Projects builds API + class Builds < Grape::API + before { authenticate! } + + resource :projects do + # Get a project builds + # + # Parameters: + # id (required) - The ID of a project + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) + # Example Request: + # GET /projects/:id/builds + get ':id/builds' do + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + + # Get builds for a specific commit of a project + # + # Parameters: + # id (required) - The ID of a project + # sha (required) - The SHA id of a commit + # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; + # if none provided showing all builds) + # Example Request: + # GET /projects/:id/repository/commits/:sha/builds + get ':id/repository/commits/:sha/builds' do + commit = user_project.ci_commits.find_by_sha(params[:sha]) + return not_found! unless commit + + builds = commit.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + + # Get a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/builds/:build_id + get ':id/builds/:build_id' do + build = get_build(params[:build_id]) + return not_found!(build) unless build + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + + # Get a trace of a specific build of a project + # + # Parameters: + # id (required) - The ID of a project + # build_id (required) - The ID of a build + # Example Request: + # GET /projects/:id/build/:build_id/trace + # + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + get ':id/builds/:build_id/trace' do + build = get_build(params[:build_id]) + return not_found!(build) unless build + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary + + trace = build.trace + body trace + end + + # Cancel a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/cancel + post ':id/builds/:build_id/cancel' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return not_found!(build) unless build + + build.cancel + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + + # Retry a specific build of a project + # + # parameters: + # id (required) - the id of a project + # build_id (required) - the id of a build + # example request: + # post /projects/:id/build/:build_id/retry + post ':id/builds/:build_id/retry' do + authorize_manage_builds! + + build = get_build(params[:build_id]) + return forbidden!('Build is not retryable') unless build && build.retryable? + + build = Ci::Build.retry(build) + + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + end + end + + helpers do + def get_build(id) + user_project.builds.find_by(id: id.to_i) + end + + def filter_builds(builds, scope) + return builds if scope.nil? || scope.empty? + + available_statuses = ::CommitStatus::AVAILABLE_STATUSES + scope = + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? + + builds.where(status: available_statuses && scope) + end + + def authorize_manage_builds! + authorize! :manage_builds, user_project + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d3803f4ec70..82a75734de0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -367,6 +367,33 @@ module API expose :id, :variables end + class Runner < Grape::Entity + expose :id + expose :description + expose :active + expose :is_shared + expose :name + end + + class Build < Grape::Entity + expose :id, :status, :stage, :name, :ref, :tag, :coverage + expose :created_at, :started_at, :finished_at + expose :user, with: User + # TODO: download_url in Ci:Build model is an GitLab Web Interface URL, not API URL. We should think on some API + # for downloading of artifacts (see: https://gitlab.com/gitlab-org/gitlab-ce/issues/4255) + expose :download_url do |repo_obj, options| + if options[:user_can_download_artifacts] + repo_obj.download_url + end + end + expose :commit, with: RepoCommit do |repo_obj, _options| + if repo_obj.respond_to?(:commit) + repo_obj.commit.commit_data + end + end + expose :runner, with: Runner + end + class Trigger < Grape::Entity expose :token, :created_at, :updated_at, :deleted_at, :last_used end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a4df810e755..6d2380cf47d 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -97,11 +97,9 @@ module API end def paginate(relation) - per_page = params[:per_page].to_i - paginated = relation.page(params[:page]).per(per_page) - add_pagination_headers(paginated, per_page) - - paginated + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end end def authenticate! @@ -289,12 +287,14 @@ module API # file helpers - def uploaded_file!(field, uploads_path) + def uploaded_file(field, uploads_path) if params[field] bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) return params[field] end + return nil unless params["#{field}.path"] && params["#{field}.name"] + # sanitize file paths # this requires all paths to exist required_attributes! %W(#{field}.path) @@ -327,16 +327,26 @@ module API private - def add_pagination_headers(paginated, per_page) + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) request_url = request.url.split('?').first links = [] - links << %(<#{request_url}?page=#{paginated.current_page - 1}&per_page=#{per_page}>; rel="prev") unless paginated.first_page? - links << %(<#{request_url}?page=#{paginated.current_page + 1}&per_page=#{per_page}>; rel="next") unless paginated.last_page? - links << %(<#{request_url}?page=1&per_page=#{per_page}>; rel="first") - links << %(<#{request_url}?page=#{paginated.total_pages}&per_page=#{per_page}>; rel="last") + links << %(<#{request_url}?page=#{paginated_data.current_page - 1}&per_page=#{paginated_data.limit_value}>; rel="prev") unless paginated_data.first_page? + links << %(<#{request_url}?page=#{paginated_data.current_page + 1}&per_page=#{paginated_data.limit_value}>; rel="next") unless paginated_data.last_page? + links << %(<#{request_url}?page=1&per_page=#{paginated_data.limit_value}>; rel="first") + links << %(<#{request_url}?page=#{paginated_data.total_pages}&per_page=#{paginated_data.limit_value}>; rel="last") - header 'Link', links.join(', ') + links.join(', ') end def abilities diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index ba2866e1efa..0257848b6bc 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Common methods for ReferenceFilters that support an optional cross-project # reference. diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index fd4fe024252..905c4c0144e 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' module Banzai module Filter diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index b2db10e6864..cdbaecf8d90 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # Issues, Merge Requests, Snippets, Commits and Commit Ranges share diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index da4ee80c1b5..856f56fb175 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index e67cd45ab9b..470727ee312 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit range references with links. diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 9e57608b483..713a56ba949 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces commit references with links. diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 86838e1483c..5952a031626 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,5 +1,4 @@ require 'action_controller' -require 'banzai' require 'gitlab_emoji' require 'html/pipeline/filter' diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 6136e73c096..edc26386903 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces external issue tracker references with links. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index ac87b9820af..8d368f3b9e7 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb new file mode 100644 index 00000000000..fe01dae4850 --- /dev/null +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -0,0 +1,151 @@ +require 'banzai' +require 'html/pipeline/filter' + +module Banzai + module Filter + # HTML Filter for parsing Gollum's tags in HTML. It's only parses the + # following tags: + # + # - Link to internal pages: + # + # * [[Bug Reports]] + # * [[How to Contribute|Contributing]] + # + # - Link to external resources: + # + # * [[http://en.wikipedia.org/wiki/Git_(software)]] + # * [[Git|http://en.wikipedia.org/wiki/Git_(software)]] + # + # - Link internal images, the special attributes will be ignored: + # + # * [[images/logo.png]] + # * [[images/logo.png|alt=Logo]] + # + # - Link external images, the special attributes will be ignored: + # + # * [[http://example.com/images/logo.png]] + # * [[http://example.com/images/logo.png|alt=Logo]] + # + # Based on Gollum::Filter::Tags + # + # Context options: + # :project_wiki (required) - Current project wiki. + # + class GollumTagsFilter < HTML::Pipeline::Filter + include ActionView::Helpers::TagHelper + + # Pattern to match tags content that should be parsed in HTML. + # + # Gollum's tags have been made to resemble the tags of other markups, + # especially MediaWiki. The basic syntax is: + # + # [[tag]] + # + # Some tags will accept attributes which are separated by pipe + # symbols.Some attributes must precede the tag and some must follow it: + # + # [[prefix-attribute|tag]] + # [[tag|suffix-attribute]] + # + # See https://github.com/gollum/gollum/wiki + # + # Rubular: http://rubular.com/r/7dQnE5CUCH + TAGS_PATTERN = %r{\[\[(.+?)\]\]} + + # Pattern to match allowed image extensions + ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i + + def call + search_text_nodes(doc).each do |node| + content = node.content + + next unless content.match(TAGS_PATTERN) + + html = process_tag($1) + + if html && html != node.content + node.replace(html) + end + end + + doc + end + + private + + # Process a single tag into its final HTML form. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML version of the tag. + def process_tag(tag) + parts = tag.split('|') + + return if parts.size.zero? + + process_image_tag(parts) || process_page_link_tag(parts) + end + + # Attempt to process the tag as an image tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid image tag or nil + # if it is not. + def process_image_tag(parts) + content = parts[0].strip + + return unless image?(content) + + if url?(content) + path = content + elsif file = project_wiki.find_file(content) + path = ::File.join project_wiki_base_path, file.path + end + + if path + content_tag(:img, nil, src: path) + end + end + + def image?(path) + path =~ ALLOWED_IMAGE_EXTENSIONS + end + + def url?(path) + path.start_with?(*%w(http https)) + end + + # Attempt to process the tag as a page link tag. + # + # tag - The String tag contents (the stuff inside the double brackets). + # + # Returns the String HTML if the tag is a valid page link tag or nil + # if it is not. + def process_page_link_tag(parts) + if parts.size == 1 + url = parts[0].strip + else + name, url = *parts.compact.map(&:strip) + end + + content_tag(:a, name || url, href: url) + end + + def project_wiki + context[:project_wiki] + end + + def project_wiki_base_path + project_wiki && project_wiki.wiki_base_path + end + + # Ensure that a :project_wiki key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project_wiki + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 51180cb901a..9f08aa36e8b 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces issue references with links. References to diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index a3a7a23c1e6..95e7d209119 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces label references with links. diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index d09cf41df39..0659fed1419 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 755b946a34b..57c71708992 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces merge request references with links. References diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 66f77902319..7141ed7c9bd 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 5dd6d2fe3c7..20bd4f7ee6e 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,5 +1,4 @@ require 'active_support/core_ext/string/output_safety' -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index bef04112919..86d484feb90 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 66f166939e4..41380627d39 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index d03e3ae4b3c..3f49d492f2f 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index 1ad5df96f85..c870a42f741 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces snippet references with links. References to diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index c889cc1e97c..8c5855e5ffc 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 9b3e67206d5..4056dcd6d64 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' module Banzai diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index d0ce13003a5..66608c9859c 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'task_list/filter' module Banzai diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 1a1d0aad8ca..f642aee0967 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline/filter' require 'uri' diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 964ab60f614..24f16f8b547 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces user or group references with links. diff --git a/lib/banzai/lazy_reference.rb b/lib/banzai/lazy_reference.rb index 073ec5d9801..1095b4debc7 100644 --- a/lib/banzai/lazy_reference.rb +++ b/lib/banzai/lazy_reference.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai class LazyReference def self.load(refs) diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb index 4e017809d9d..142a9962eb1 100644 --- a/lib/banzai/pipeline.rb +++ b/lib/banzai/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline def self.[](name) diff --git a/lib/banzai/pipeline/asciidoc_pipeline.rb b/lib/banzai/pipeline/asciidoc_pipeline.rb index 5e76a817be5..f1331c0ebf9 100644 --- a/lib/banzai/pipeline/asciidoc_pipeline.rb +++ b/lib/banzai/pipeline/asciidoc_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AsciidocPipeline < BasePipeline diff --git a/lib/banzai/pipeline/atom_pipeline.rb b/lib/banzai/pipeline/atom_pipeline.rb index 957f352aec5..9694e4bc23f 100644 --- a/lib/banzai/pipeline/atom_pipeline.rb +++ b/lib/banzai/pipeline/atom_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class AtomPipeline < FullPipeline diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index cd30009e5c0..db5177db7b3 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,4 +1,3 @@ -require 'banzai' require 'html/pipeline' module Banzai diff --git a/lib/banzai/pipeline/combined_pipeline.rb b/lib/banzai/pipeline/combined_pipeline.rb index f3bf1809d18..9485199132e 100644 --- a/lib/banzai/pipeline/combined_pipeline.rb +++ b/lib/banzai/pipeline/combined_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline module CombinedPipeline diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index 94c2cb165a5..20e24ace352 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class DescriptionPipeline < FullPipeline diff --git a/lib/banzai/pipeline/email_pipeline.rb b/lib/banzai/pipeline/email_pipeline.rb index 14356145a35..e47c384afc1 100644 --- a/lib/banzai/pipeline/email_pipeline.rb +++ b/lib/banzai/pipeline/email_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class EmailPipeline < FullPipeline diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index 72395a5d50e..d47ddfda4be 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 838155e8831..b7a38ea8427 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class GfmPipeline < BasePipeline diff --git a/lib/banzai/pipeline/note_pipeline.rb b/lib/banzai/pipeline/note_pipeline.rb index 89335143852..7890f20f716 100644 --- a/lib/banzai/pipeline/note_pipeline.rb +++ b/lib/banzai/pipeline/note_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class NotePipeline < FullPipeline diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 998fd75daa2..3fbc681457b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 148f24b6ce1..bd338c045f3 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class PostProcessPipeline < BasePipeline diff --git a/lib/banzai/pipeline/reference_extraction_pipeline.rb b/lib/banzai/pipeline/reference_extraction_pipeline.rb index 4f9bc9fcccc..eaddccd30a5 100644 --- a/lib/banzai/pipeline/reference_extraction_pipeline.rb +++ b/lib/banzai/pipeline/reference_extraction_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class ReferenceExtractionPipeline < BasePipeline diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index a3c9d4f43aa..8b84ab401df 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class SingleLinePipeline < GfmPipeline diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb new file mode 100644 index 00000000000..50b5450e70b --- /dev/null +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -0,0 +1,11 @@ +require 'banzai' + +module Banzai + module Pipeline + class WikiPipeline < FullPipeline + def self.filters + super.insert(1, Filter::GollumTagsFilter) + end + end + end +end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 2c197d31898..f4079538ec5 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 15faa6edd84..fb87637b94f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -78,11 +78,13 @@ module Ci # Parameters: # id (required) - The ID of a build # token (required) - The build authorization token - # file (required) - The uploaded file + # file (required) - Artifacts file # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition # file.type - real content type as send in Content-Type + # metadata.path - path to locally stored body (generated by Workhorse) + # metadata.name - filename (generated by Workhorse) # Headers: # BUILD-TOKEN (required) - The build authorization token, the same as token # Body: @@ -96,13 +98,20 @@ module Ci build = Ci::Build.find_by_id(params[:id]) not_found! unless build authenticate_build_token!(build) - forbidden!('build is not running') unless build.running? + forbidden!('Build is not running!') unless build.running? - file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) - file_to_large! unless file.size < max_artifacts_size + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) - if build.update_attributes(artifacts_file: file) - present build, with: Entities::Build + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + build.artifacts_file = artifacts + build.artifacts_metadata = metadata + + if build.save + present(build, with: Entities::Build) else render_validation_error!(build) end @@ -148,6 +157,7 @@ module Ci not_found! unless build authenticate_build_token!(build) build.remove_artifacts_file! + build.remove_artifacts_metadata! end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb new file mode 100644 index 00000000000..1344f5d120b --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -0,0 +1,109 @@ +require 'zlib' +require 'json' + +module Gitlab + module Ci + module Build + module Artifacts + class Metadata + class ParserError < StandardError; end + + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} + + attr_reader :file, :path, :full_version + + def initialize(file, path) + @file, @path = file, path + @full_version = read_version + end + + def version + @full_version.match(VERSION_PATTERN)[1] + end + + def errors + gzip do |gz| + read_string(gz) # version + errors = read_string(gz) + raise ParserError, 'Errors field not found!' unless errors + + begin + JSON.parse(errors) + rescue JSON::ParserError + raise ParserError, 'Invalid errors field!' + end + end + end + + def find_entries! + gzip do |gz| + 2.times { read_string(gz) } # version and errors fields + match_entries(gz) + end + end + + def to_entry + entries = find_entries! + Entry.new(@path, entries) + end + + private + + def match_entries(gz) + entries = {} + match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$} + + until gz.eof? do + begin + path = read_string(gz).force_encoding('UTF-8') + meta = read_string(gz).force_encoding('UTF-8') + + next unless path.valid_encoding? && meta.valid_encoding? + next unless path =~ match_pattern + next if path =~ INVALID_PATH_PATTERN + + entries[path] = JSON.parse(meta, symbolize_names: true) + rescue JSON::ParserError, Encoding::CompatibilityError + next + end + end + + entries + end + + def read_version + gzip do |gz| + version_string = read_string(gz) + + unless version_string + raise ParserError, 'Artifacts metadata file empty!' + end + + unless version_string =~ VERSION_PATTERN + raise ParserError, 'Invalid version!' + end + + version_string.chomp + end + end + + def read_uint32(gz) + binary = gz.read(4) + binary.unpack('L>')[0] if binary + end + + def read_string(gz) + string_size = read_uint32(gz) + return nil unless string_size + gz.read(string_size) + end + + def gzip(&block) + Zlib::GzipReader.open(@file, &block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb new file mode 100644 index 00000000000..25b71fc3275 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci::Build::Artifacts + class Metadata + ## + # Class that represents an entry (path and metadata) to a file or + # directory in GitLab CI Build Artifacts binary file / archive + # + # This is IO-operations safe class, that does similar job to + # Ruby's Pathname but without the risk of accessing filesystem. + # + # This class is working only with UTF-8 encoded paths. + # + class Entry + attr_reader :path, :entries + attr_accessor :name + + def initialize(path, entries) + @path = path.dup.force_encoding('UTF-8') + @entries = entries + + if path.include?("\0") + raise ArgumentError, 'Path contains zero byte character!' + end + + unless path.valid_encoding? + raise ArgumentError, 'Path contains non-UTF-8 byte sequence!' + end + end + + def directory? + blank_node? || @path.end_with?('/') + end + + def file? + !directory? + end + + def has_parent? + nodes > 0 + end + + def parent + return nil unless has_parent? + self.class.new(@path.chomp(basename), @entries) + end + + def basename + (directory? && !blank_node?) ? name + '/' : name + end + + def name + @name || @path.split('/').last.to_s + end + + def children + return [] unless directory? + return @children if @children + + child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$} + @children = select_entries { |path| path =~ child_pattern } + end + + def directories(opts = {}) + return [] unless directory? + dirs = children.select(&:directory?) + return dirs unless has_parent? && opts[:parent] + + dotted_parent = parent + dotted_parent.name = '..' + dirs.prepend(dotted_parent) + end + + def files + return [] unless directory? + children.select(&:file?) + end + + def metadata + @entries[@path] || {} + end + + def nodes + @path.count('/') + (file? ? 1 : 0) + end + + def blank_node? + @path.empty? # "" is considered to be './' + end + + def exists? + blank_node? || @entries.include?(@path) + end + + def empty? + children.empty? + end + + def to_s + @path + end + + def ==(other) + @path == other.path && @entries == other.entries + end + + def inspect + "#{self.class.name}: #{@path}" + end + + private + + def select_entries + selected = @entries.select { |path, _metadata| yield path } + selected.map { |path, _metadata| self.class.new(path, @entries) } + end + end + end + end +end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7015fe36c3d..516e59b87a3 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -56,8 +56,9 @@ module Gitlab private def filename?(line) - line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b', - '--- /tmp/diffy', '+++ /tmp/diffy') + line.start_with?( '--- /dev/null', '+++ /dev/null', '--- a', '+++ b', + '+++ a', # The line will start with `+++ a` in the reverse diff of an orphan commit + '--- /tmp/diffy', '+++ /tmp/diffy') end def identification_type(line) diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2b0afbc7b39..18929b9113b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class Importer + include Gitlab::ShellAdapter + attr_reader :project, :client def initialize(project) @@ -12,10 +14,7 @@ module Gitlab end def execute - import_issues - import_pull_requests - - true + import_issues && import_pull_requests && import_wiki end private @@ -34,6 +33,10 @@ module Gitlab end end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_pull_requests @@ -48,6 +51,10 @@ module Gitlab import_comments_on_diff(pull_request.number, merge_request) end end + + true + rescue ActiveRecord::RecordInvalid + false end def import_comments(issue_number, noteable) @@ -66,6 +73,18 @@ module Gitlab noteable.notes.create!(comment.attributes) end end + + def import_wiki + unless project.wiki_enabled? + wiki = WikiFormatter.new(project) + gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + project.update_attribute(:wiki_enabled, true) + end + + true + rescue Gitlab::Shell::Error + false + end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 8c27ebd1ce8..474927069a5 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -20,7 +20,8 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), + wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later ).execute project.create_import_data(data: { "github_session" => session_data } ) diff --git a/lib/gitlab/github_import/wiki_formatter.rb b/lib/gitlab/github_import/wiki_formatter.rb new file mode 100644 index 00000000000..6c592ff469c --- /dev/null +++ b/lib/gitlab/github_import/wiki_formatter.rb @@ -0,0 +1,19 @@ +module Gitlab + module GithubImport + class WikiFormatter + attr_reader :project + + def initialize(project) + @project = project + end + + def path_with_namespace + "#{project.path_with_namespace}.wiki" + end + + def import_url + project.import_url.sub(/\.git\z/, ".wiki.git") + end + end + end +end diff --git a/lib/gitlab/markdown/pipeline.rb b/lib/gitlab/markdown/pipeline.rb index 8f3f43c0e91..699d8b9fc07 100644 --- a/lib/gitlab/markdown/pipeline.rb +++ b/lib/gitlab/markdown/pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab module Markdown class Pipeline diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 4164e998dd1..4d830aa45e1 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor |