diff options
author | Alfredo Sumaran <alfredo@gitlab.com> | 2016-04-20 20:24:42 +0300 |
---|---|---|
committer | Alfredo Sumaran <alfredo@gitlab.com> | 2016-04-20 20:24:42 +0300 |
commit | a45924348fca7eff4e26b3d9e47117b9fb3669ec (patch) | |
tree | 17531b97fd4a26e4f530dc5d35c2962fc0ed4f06 /lib | |
parent | b41ddf9c6977e311b5085de8467ab450aa81b71b (diff) | |
parent | 832cdd3d516698d0e6a7257b3d94292819a0436a (diff) |
Merge remote-tracking branch 'origin/master' into issue_14904
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/api.rb | 1 | ||||
-rw-r--r-- | lib/api/entities.rb | 12 | ||||
-rw-r--r-- | lib/api/licenses.rb | 58 | ||||
-rw-r--r-- | lib/api/tags.rb | 2 | ||||
-rw-r--r-- | lib/ci/api/api.rb | 2 | ||||
-rw-r--r-- | lib/ci/api/builds.rb | 33 | ||||
-rw-r--r-- | lib/ci/gitlab_ci_yaml_processor.rb | 75 | ||||
-rw-r--r-- | lib/file_size_validator.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/github_import/issue_formatter.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/github_import/label_formatter.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/github_import/milestone_formatter.rb | 48 | ||||
-rw-r--r-- | lib/gitlab/github_import/pull_request_formatter.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/metrics.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/push_data_builder.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/url_builder.rb | 6 |
16 files changed, 289 insertions, 40 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 7d65145176b..cc1004f8005 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -57,5 +57,6 @@ module API mount Builds mount Variables mount Runners + mount Licenses end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 60b9f5e0ece..716ca6f7ed9 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -439,5 +439,17 @@ module API class Variable < Grape::Entity expose :key, :value end + + class RepoLicense < Grape::Entity + expose :key, :name, :nickname + expose :featured, as: :popular + expose :url, as: :html_url + expose(:source_url) { |license| license.meta['source'] } + expose(:description) { |license| license.meta['description'] } + expose(:conditions) { |license| license.meta['conditions'] } + expose(:permissions) { |license| license.meta['permissions'] } + expose(:limitations) { |license| license.meta['limitations'] } + expose :content + end end end diff --git a/lib/api/licenses.rb b/lib/api/licenses.rb new file mode 100644 index 00000000000..187d2c04703 --- /dev/null +++ b/lib/api/licenses.rb @@ -0,0 +1,58 @@ +module API + # Licenses API + class Licenses < Grape::API + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + + # Get the list of the available license templates + # + # Parameters: + # popular - Filter licenses to only the popular ones + # + # Example Request: + # GET /licenses + # GET /licenses?popular=1 + get 'licenses' do + options = { + featured: params[:popular].present? ? true : nil + } + present Licensee::License.all(options), with: Entities::RepoLicense + end + + # Get text for specific license + # + # Parameters: + # key (required) - The key of a license + # project - Copyrighted project name + # fullname - Full name of copyright holder + # + # Example Request: + # GET /licenses/mit + # + get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do + required_attributes! [:key] + + not_found!('License') unless Licensee::License.find(params[:key]) + + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + license = Licensee::License.new(params[:key]) + + license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + + present license, with: Entities::RepoLicense + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index d1a10479e44..3e1ed3fe5c7 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -12,7 +12,7 @@ module API # Example Request: # GET /projects/:id/repository/tags get ":id/repository/tags" do - present user_project.repo.tags.sort_by(&:name).reverse, + present user_project.repository.tags.sort_by(&:name).reverse, with: Entities::RepoTag, project: user_project end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 4e85d2c3c74..353c4ddebf8 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -23,6 +23,8 @@ module Ci rack_response({ 'message' => '500 Internal Server Error' }, 500) end + content_type :txt, 'text/plain' + content_type :json, 'application/json' format :json helpers ::Ci::API::Helpers diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 2e9a5d311f9..607359769d1 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -50,6 +50,39 @@ module Ci end end + # Send incremental log update - Runners only + # + # Parameters: + # id (required) - The ID of a build + # Body: + # content of logs to append + # Headers: + # Content-Range (required) - range of content that was sent + # BUILD-TOKEN (required) - The build authorization token + # Example Request: + # PATCH /builds/:id/trace.txt + patch ":id/trace.txt" do + build = Ci::Build.find_by_id(params[:id]) + not_found! unless build + authenticate_build_token!(build) + forbidden!('Build has been erased!') if build.erased? + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = build.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + build.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Build-Status', build.status + header 'Range', "0-#{build.trace_length}" + end + # Authorize artifacts uploading for build - Runners only # # Parameters: diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b8ede3a7edc..ff9887cba1e 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,12 +4,12 @@ module Ci DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' - ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables, :cache] + ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :variables] + :dependencies, :before_script, :after_script, :variables] - attr_reader :before_script, :image, :services, :path, :cache + attr_reader :before_script, :after_script, :image, :services, :path, :cache def initialize(config, path = nil) @config = YAML.safe_load(config, [Symbol], [], true) @@ -55,6 +55,7 @@ module Ci def initial_parsing @before_script = @config[:before_script] || [] + @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @stages = @config[:stages] || @config[:types] @@ -83,7 +84,7 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], @@ -96,23 +97,32 @@ module Ci artifacts: job[:artifacts], cache: job[:cache] || @cache, dependencies: job[:dependencies], + after_script: job[:after_script] || @after_script, }.compact } end - def normalize_script(script) - if script.is_a? Array - script.join("\n") - else - script + def validate! + validate_global! + + @jobs.each do |name, job| + validate_job!(name, job) end + + true end - def validate! + private + + def validate_global! unless validate_array_of_strings(@before_script) raise ValidationError, "before_script should be an array of strings" end + unless @after_script.nil? || validate_array_of_strings(@after_script) + raise ValidationError, "after_script should be an array of strings" + end + unless @image.nil? || @image.is_a?(String) raise ValidationError, "image should be a string" end @@ -129,31 +139,28 @@ module Ci raise ValidationError, "variables should be a map of key-value strings" end - if @cache - if @cache[:key] && !validate_string(@cache[:key]) - raise ValidationError, "cache:key parameter should be a string" - end - - if @cache[:untracked] && !validate_boolean(@cache[:untracked]) - raise ValidationError, "cache:untracked parameter should be an boolean" - end + validate_global_cache! if @cache + end - if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) - raise ValidationError, "cache:paths parameter should be an array of strings" - end + def validate_global_cache! + if @cache[:key] && !validate_string(@cache[:key]) + raise ValidationError, "cache:key parameter should be a string" end - @jobs.each do |name, job| - validate_job!(name, job) + if @cache[:untracked] && !validate_boolean(@cache[:untracked]) + raise ValidationError, "cache:untracked parameter should be an boolean" end - true + if @cache[:paths] && !validate_array_of_strings(@cache[:paths]) + raise ValidationError, "cache:paths parameter should be an array of strings" + end end def validate_job!(name, job) validate_job_name!(name) validate_job_keys!(name, job) validate_job_types!(name, job) + validate_job_script!(name, job) validate_job_stage!(name, job) if job[:stage] validate_job_variables!(name, job) if job[:variables] @@ -162,8 +169,6 @@ module Ci validate_job_dependencies!(name, job) if job[:dependencies] end - private - def validate_job_name!(name) if name.blank? || !validate_string(name) raise ValidationError, "job name should be non-empty string" @@ -179,10 +184,6 @@ module Ci end def validate_job_types!(name, job) - if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) - raise ValidationError, "#{name} job: script should be a string or an array of a strings" - end - if job[:image] && !validate_string(job[:image]) raise ValidationError, "#{name} job: image should be a string" end @@ -212,6 +213,20 @@ module Ci end end + def validate_job_script!(name, job) + if !validate_string(job[:script]) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name} job: script should be a string or an array of a strings" + end + + if job[:before_script] && !validate_array_of_strings(job[:before_script]) + raise ValidationError, "#{name} job: before_script should be an array of strings" + end + + if job[:after_script] && !validate_array_of_strings(job[:after_script]) + raise ValidationError, "#{name} job: after_script should be an array of strings" + end + end + def validate_job_stage!(name, job) unless job[:stage].is_a?(String) && job[:stage].in?(stages) raise ValidationError, "#{name} job: stage parameter should be #{stages.join(", ")}" diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 2eae55e534b..440dd44ece7 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -1,9 +1,9 @@ class FileSizeValidator < ActiveModel::EachValidator - MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze - CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze + MESSAGES = { is: :wrong_size, minimum: :size_too_small, maximum: :size_too_big }.freeze + CHECKS = { is: :==, minimum: :>=, maximum: :<= }.freeze - DEFAULT_TOKENIZER = lambda { |value| value.split(//) } - RESERVED_OPTIONS = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long] + DEFAULT_TOKENIZER = -> (value) { value.split(//) }.freeze + RESERVED_OPTIONS = [:minimum, :maximum, :within, :is, :tokenizer, :too_short, :too_long].freeze def initialize(options) if range = (options.delete(:in) || options.delete(:within)) diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 0b1ed510229..0f9e3ee14ee 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -16,7 +16,8 @@ module Gitlab end def execute - import_issues && import_pull_requests && import_wiki + import_labels && import_milestones && import_issues && + import_pull_requests && import_wiki end private @@ -25,6 +26,26 @@ module Gitlab @import_data_credentials ||= project.import_data.credentials if project.import_data end + def import_labels + client.labels(project.import_source).each do |raw_data| + Label.create!(LabelFormatter.new(project, raw_data).attributes) + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + + def import_milestones + client.list_milestones(project.import_source, state: :all).each do |raw_data| + Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes) + end + + true + rescue ActiveRecord::RecordInvalid => e + raise Projects::ImportService::Error, e.message + end + def import_issues client.list_issues(project.import_source, state: :all, sort: :created, @@ -33,6 +54,7 @@ module Gitlab if gh_issue.valid? issue = Issue.create!(gh_issue.attributes) + apply_labels(gh_issue.number, issue) if gh_issue.has_comments? import_comments(gh_issue.number, issue) @@ -55,6 +77,7 @@ module Gitlab merge_request = MergeRequest.new(pull_request.attributes) if merge_request.save + apply_labels(pull_request.number, merge_request) import_comments(pull_request.number, merge_request) import_comments_on_diff(pull_request.number, merge_request) end @@ -66,6 +89,18 @@ module Gitlab raise Projects::ImportService::Error, e.message end + def apply_labels(number, issuable) + issue = client.issue(project.import_source, number) + + if issue.labels.count > 0 + label_ids = issue.labels.map do |raw| + Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) + end + + issuable.update_attribute(:label_ids, label_ids) + end + end + def import_comments(issue_number, noteable) comments = client.issue_comments(project.import_source, issue_number) create_comments(comments, noteable) diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index acb332cb0cb..c8173913b4e 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -5,6 +5,7 @@ module Gitlab { iid: number, project: project, + milestone: milestone, title: raw_data.title, description: description, state: state, @@ -55,6 +56,12 @@ module Gitlab @formatter.author_line(author) + body end + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end + end + def state raw_data.state == 'closed' ? 'closed' : 'opened' end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb new file mode 100644 index 00000000000..c2b9d40b511 --- /dev/null +++ b/lib/gitlab/github_import/label_formatter.rb @@ -0,0 +1,23 @@ +module Gitlab + module GithubImport + class LabelFormatter < BaseFormatter + def attributes + { + project: project, + title: title, + color: color + } + end + + private + + def color + "##{raw_data.color}" + end + + def title + raw_data.name + end + end + end +end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb new file mode 100644 index 00000000000..e91a7e328cf --- /dev/null +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -0,0 +1,48 @@ +module Gitlab + module GithubImport + class MilestoneFormatter < BaseFormatter + def attributes + { + iid: number, + project: project, + title: title, + description: description, + due_date: due_date, + state: state, + created_at: created_at, + updated_at: updated_at + } + end + + private + + def number + raw_data.number + end + + def title + raw_data.title + end + + def description + raw_data.description + end + + def due_date + raw_data.due_on + end + + def state + raw_data.state == 'closed' ? 'closed' : 'active' + end + + def created_at + raw_data.created_at + end + + def updated_at + state == 'closed' ? raw_data.closed_at : raw_data.updated_at + end + end + end +end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 5ee8a14624a..d21b942ad4b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -11,6 +11,7 @@ module Gitlab target_project: target_project, target_branch: target_branch.name, state: state, + milestone: milestone, author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, @@ -58,6 +59,12 @@ module Gitlab formatter.author_line(author) + body end + def milestone + if raw_data.milestone.present? + project.milestones.find_by(iid: raw_data.milestone.number) + end + end + def source_project project end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 484970c5a10..f82dce14865 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -14,7 +14,8 @@ module Gitlab method_call_threshold: current_application_settings[:metrics_method_call_threshold], host: current_application_settings[:metrics_host], port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15 + sample_interval: current_application_settings[:metrics_sample_interval] || 15, + packet_size: current_application_settings[:metrics_packet_size] || 1 } end @@ -41,9 +42,9 @@ module Gitlab prepared = prepare_metrics(metrics) pool.with do |connection| - prepared.each do |metric| + prepared.each_slice(settings[:packet_size]) do |slice| begin - connection.write_points([metric]) + connection.write_points(slice) rescue StandardError end end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/push_data_builder.rb index 97d1edab9c1..67622f321a6 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/push_data_builder.rb @@ -36,11 +36,12 @@ module Gitlab commit.hook_attrs(with_changed_files: true) end - type = Gitlab::Git.tag_ref?(ref) ? "tag_push" : "push" + type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data data = { object_kind: type, + event_name: type, before: oldrev, after: newrev, ref: ref, diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f1943222edf..2bbbd3074e8 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -20,6 +20,8 @@ module Gitlab merge_request_url(object) when Note note_url + when WikiPage + wiki_page_url else raise NotImplementedError.new("No URL builder defined for #{object.class}") end @@ -58,5 +60,9 @@ module Gitlab project_snippet_url(snippet, anchor: dom_id(object)) end end + + def wiki_page_url + "#{Gitlab.config.gitlab.url}#{object.wiki.wiki_base_path}/#{object.slug}" + end end end |