diff options
Diffstat (limited to 'lib')
88 files changed, 2801 insertions, 323 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index eebd44ea5b6..c09488d3547 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -50,5 +50,6 @@ module API mount Branches mount Labels mount Settings + mount Keys end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1f9dd6bc152..33b6224a810 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -8,7 +8,7 @@ module API expose :id, :state, :avatar_url expose :web_url do |user, options| - Rails.application.routes.url_helpers.user_url(user) + Gitlab::Application.routes.url_helpers.user_url(user) end end @@ -81,7 +81,7 @@ module API expose :avatar_url expose :web_url do |group, options| - Rails.application.routes.url_helpers.group_url(group) + Gitlab::Application.routes.url_helpers.group_url(group) end end @@ -199,6 +199,10 @@ module API expose :id, :title, :key, :created_at end + class SSHKeyWithUser < SSHKey + expose :user, using: Entities::UserFull + end + class Note < Grape::Entity expose :id expose :note, as: :body diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1ebf9a1f022..7fada98fcdc 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -55,6 +55,32 @@ module API end end + def project_service + @project_service ||= begin + underscored_service = params[:service_slug].underscore + + if Service.available_services_names.include?(underscored_service) + user_project.build_missing_services + + service_method = "#{underscored_service}_service" + + send_service(service_method) + end + end + + @project_service || not_found!("Service") + end + + def send_service(service_method) + user_project.send(service_method) + end + + def service_attributes + @service_attributes ||= project_service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym + end + end + def find_group(id) begin group = Group.find(id) @@ -122,15 +148,14 @@ module API end end - def attributes_for_keys(keys) + def attributes_for_keys(keys, custom_params = nil) + params_hash = custom_params || params attrs = {} - keys.each do |key| if params[key].present? or (params.has_key?(key) and params[key] == false) attrs[key] = params[key] end end - ActionController::Parameters.new(attrs).permit! end @@ -220,6 +245,44 @@ module API error!({ 'message' => message }, status) end + # Projects helpers + + def filter_projects(projects) + # If the archived parameter is passed, limit results accordingly + if params[:archived].present? + projects = projects.where(archived: parse_boolean(params[:archived])) + end + + if params[:search].present? + projects = projects.search(params[:search]) + end + + if params[:ci_enabled_first].present? + projects.includes(:gitlab_ci_service). + reorder("services.active DESC, projects.#{project_order_by} #{project_sort}") + else + projects.reorder(project_order_by => project_sort) + end + end + + def project_order_by + order_fields = %w(id name path created_at updated_at last_activity_at) + + if order_fields.include?(params['order_by']) + params['order_by'] + else + 'created_at' + end + end + + def project_sort + if params["sort"] == 'asc' + :asc + else + :desc + end + end + private def add_pagination_headers(paginated, per_page) diff --git a/lib/api/keys.rb b/lib/api/keys.rb new file mode 100644 index 00000000000..2b723b79504 --- /dev/null +++ b/lib/api/keys.rb @@ -0,0 +1,20 @@ +module API + # Keys API + class Keys < Grape::API + before { authenticate! } + + resource :keys do + # Get single ssh key by id. Only available to admin users. + # + # Example Request: + # GET /keys/:id + get ":id" do + authenticated_as_admin! + + key = Key.find(params[:id]) + + present key, with: Entities::SSHKeyWithUser + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 7412274b045..63ea2f05438 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -55,7 +55,7 @@ module API else merge_requests end - merge_requests.reorder(issuable_order_by => issuable_sort) + merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) present paginate(merge_requests), with: Entities::MergeRequest end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 1f2251c9b9c..c2fb36b4143 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,42 +11,6 @@ module API attrs[:visibility_level] = Gitlab::VisibilityLevel::PUBLIC if !attrs[:visibility_level].present? && publik == true attrs end - - def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: parse_boolean(params[:archived])) - end - - if params[:search].present? - projects = projects.search(params[:search]) - end - - if params[:ci_enabled_first].present? - projects.includes(:gitlab_ci_service). - reorder("services.active DESC, projects.#{project_order_by} #{project_sort}") - else - projects.reorder(project_order_by => project_sort) - end - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end end # Get a projects list for authenticated user diff --git a/lib/api/services.rb b/lib/api/services.rb index 3ad59cf3adf..6727e80ac1e 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -4,74 +4,60 @@ module API before { authenticate! } before { authorize_admin_project } + resource :projects do - # Set GitLab CI service for project - # - # Parameters: - # token (required) - CI project token - # project_url (required) - CI project url + # Set <service_slug> service for project # # Example Request: + # # PUT /projects/:id/services/gitlab-ci - put ":id/services/gitlab-ci" do - required_attributes! [:token, :project_url] - attrs = attributes_for_keys [:token, :project_url] - user_project.build_missing_services + # + put ':id/services/:service_slug' do + if project_service + validators = project_service.class.validators.select do |s| + s.class == ActiveRecord::Validations::PresenceValidator && + s.attributes != [:project_id] + end + + required_attributes! validators.map(&:attributes).flatten.uniq + attrs = attributes_for_keys service_attributes - if user_project.gitlab_ci_service.update_attributes(attrs.merge(active: true)) - true - else - not_found! + if project_service.update_attributes(attrs.merge(active: true)) + true + else + not_found! + end end end - # Delete GitLab CI service settings + # Delete <service_slug> service for project # # Example Request: - # DELETE /projects/:id/services/gitlab-ci - delete ":id/services/gitlab-ci" do - if user_project.gitlab_ci_service - user_project.gitlab_ci_service.update_attributes( - active: false, - token: nil, - project_url: nil - ) - end - end - - # Set Hipchat service for project # - # Parameters: - # token (required) - Hipchat token - # room (required) - Hipchat room name + # DELETE /project/:id/services/gitlab-ci # - # Example Request: - # PUT /projects/:id/services/hipchat - put ':id/services/hipchat' do - required_attributes! [:token, :room] - attrs = attributes_for_keys [:token, :room] - user_project.build_missing_services + delete ':id/services/:service_slug' do + if project_service + attrs = service_attributes.inject({}) do |hash, key| + hash.merge!(key => nil) + end - if user_project.hipchat_service.update_attributes( - attrs.merge(active: true)) - true - else - not_found! + if project_service.update_attributes(attrs.merge(active: false)) + true + else + not_found! + end end end - # Delete Hipchat service settings + # Get <service_slug> service settings for project # # Example Request: - # DELETE /projects/:id/services/hipchat - delete ':id/services/hipchat' do - if user_project.hipchat_service - user_project.hipchat_service.update_attributes( - active: false, - token: nil, - room: nil - ) - end + # + # GET /project/:id/services/gitlab-ci + # + get ':id/services/:service_slug' do + present project_service end end end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb new file mode 100644 index 00000000000..6f56f680bb9 --- /dev/null +++ b/lib/backup/builds.rb @@ -0,0 +1,34 @@ +module Backup + class Builds + attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + + def initialize + @app_builds_dir = Settings.gitlab_ci.builds_path + @backup_dir = Gitlab.config.backup.path + @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') + end + + # Copy builds from builds directory to backup/builds + def dump + FileUtils.rm_rf(backup_builds_dir) + # Ensure the parent dir of backup_builds_dir exists + FileUtils.mkdir_p(Gitlab.config.backup.path) + # Fail if somebody raced to create backup_builds_dir before us + FileUtils.mkdir(backup_builds_dir, mode: 0700) + FileUtils.cp_r(app_builds_dir, backup_dir) + end + + def restore + backup_existing_builds_dir + + FileUtils.cp_r(backup_builds_dir, app_builds_dir) + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 13c68d9354f..ac63f89c6ec 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -153,7 +153,7 @@ module Backup end def folders_to_backup - folders = %w{repositories db uploads} + folders = %w{repositories db uploads builds} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb new file mode 100644 index 00000000000..ac6d667cf8d --- /dev/null +++ b/lib/ci/ansi2html.rb @@ -0,0 +1,224 @@ +# ANSI color library +# +# Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code +module Ci + module Ansi2html + # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) + COLOR = { + 0 => 'black', # not that this is gray in the intense color table + 1 => 'red', + 2 => 'green', + 3 => 'yellow', + 4 => 'blue', + 5 => 'magenta', + 6 => 'cyan', + 7 => 'white', # not that this is gray in the dark (aka default) color table + } + + STYLE_SWITCHES = { + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10, + } + + def self.convert(ansi) + Converter.new().convert(ansi) + end + + class Converter + def on_0(s) reset() end + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_9(s) enable(STYLE_SWITCHES[:cross]) end + + def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_29(s) disable(STYLE_SWITCHES[:cross]) end + + def on_30(s) set_fg_color(0) end + def on_31(s) set_fg_color(1) end + def on_32(s) set_fg_color(2) end + def on_33(s) set_fg_color(3) end + def on_34(s) set_fg_color(4) end + def on_35(s) set_fg_color(5) end + def on_36(s) set_fg_color(6) end + def on_37(s) set_fg_color(7) end + def on_38(s) set_fg_color_256(s) end + def on_39(s) set_fg_color(9) end + + def on_40(s) set_bg_color(0) end + def on_41(s) set_bg_color(1) end + def on_42(s) set_bg_color(2) end + def on_43(s) set_bg_color(3) end + def on_44(s) set_bg_color(4) end + def on_45(s) set_bg_color(5) end + def on_46(s) set_bg_color(6) end + def on_47(s) set_bg_color(7) end + def on_48(s) set_bg_color_256(s) end + def on_49(s) set_bg_color(9) end + + def on_90(s) set_fg_color(0, 'l') end + def on_91(s) set_fg_color(1, 'l') end + def on_92(s) set_fg_color(2, 'l') end + def on_93(s) set_fg_color(3, 'l') end + def on_94(s) set_fg_color(4, 'l') end + def on_95(s) set_fg_color(5, 'l') end + def on_96(s) set_fg_color(6, 'l') end + def on_97(s) set_fg_color(7, 'l') end + def on_99(s) set_fg_color(9, 'l') end + + def on_100(s) set_bg_color(0, 'l') end + def on_101(s) set_bg_color(1, 'l') end + def on_102(s) set_bg_color(2, 'l') end + def on_103(s) set_bg_color(3, 'l') end + def on_104(s) set_bg_color(4, 'l') end + def on_105(s) set_bg_color(5, 'l') end + def on_106(s) set_bg_color(6, 'l') end + def on_107(s) set_bg_color(7, 'l') end + def on_109(s) set_bg_color(9, 'l') end + + def convert(ansi) + @out = "" + @n_open_tags = 0 + reset() + + s = StringScanner.new(ansi.gsub("<", "<")) + while(!s.eos?) + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + else + @out << s.scan(/./m) + end + end + + close_open_tags() + @out + end + + def handle_sequence(s) + indicator = s[1] + commands = s[2].split ';' + terminator = s[3] + + # We are only interested in color and text style changes - triggered by + # sequences starting with '\e[' and ending with 'm'. Any other control + # sequence gets stripped (including stuff like "delete last line") + return unless indicator == '[' and terminator == 'm' + + close_open_tags() + + if commands.empty?() + reset() + return + end + + evaluate_command_stack(commands) + + css_classes = [] + + unless @fg_color.nil? + fg_color = @fg_color + # Most terminals show bold colored text in the light color variant + # Let's mimic that here + if @style_mask & STYLE_SWITCHES[:bold] != 0 + fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + end + css_classes << fg_color + end + css_classes << @bg_color unless @bg_color.nil? + + STYLE_SWITCHES.each do |css_class, flag| + css_classes << "term-#{css_class}" if @style_mask & flag != 0 + end + + open_new_tag(css_classes) if css_classes.length > 0 + end + + def evaluate_command_stack(stack) + return unless command = stack.shift() + + if self.respond_to?("on_#{command}", true) + self.send("on_#{command}", stack) + end + + evaluate_command_stack(stack) + end + + def open_new_tag(css_classes) + @out << %{<span class="#{css_classes.join(' ')}">} + @n_open_tags += 1 + end + + def close_open_tags + while @n_open_tags > 0 + @out << %{</span>} + @n_open_tags -= 1 + end + end + + def reset + @fg_color = nil + @bg_color = nil + @style_mask = 0 + end + + def enable(flag) + @style_mask |= flag + end + + def disable(flag) + @style_mask &= ~flag + end + + def set_fg_color(color_index, prefix = nil) + @fg_color = get_term_color_class(color_index, ["fg", prefix]) + end + + def set_bg_color(color_index, prefix = nil) + @bg_color = get_term_color_class(color_index, ["bg", prefix]) + end + + def get_term_color_class(color_index, prefix) + color_name = COLOR[color_index] + return nil if color_name.nil? + + get_color_class(["term", prefix, color_name]) + end + + def set_fg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "fg") + @fg_color = css_class unless css_class.nil? + end + + def set_bg_color_256(command_stack) + css_class = get_xterm_color_class(command_stack, "bg") + @bg_color = css_class unless css_class.nil? + end + + def get_xterm_color_class(command_stack, prefix) + # the 38 and 48 commands have to be followed by "5" and the color index + return unless command_stack.length >= 2 + return unless command_stack[0] == "5" + + command_stack.shift() # ignore the "5" command + color_index = command_stack.shift().to_i + + return unless color_index >= 0 + return unless color_index <= 255 + + get_color_class(["xterm", prefix, color_index]) + end + + def get_color_class(segments) + [segments].flatten.compact.join('-') + end + end + end +end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb new file mode 100644 index 00000000000..172c6f22164 --- /dev/null +++ b/lib/ci/api/api.rb @@ -0,0 +1,39 @@ +Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file} + +module Ci + module API + class API < Grape::API + include APIGuard + version 'v1', using: :path + + rescue_from ActiveRecord::RecordNotFound do + rack_response({ 'message' => '404 Not found' }.to_json, 404) + end + + rescue_from :all do |exception| + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + # why is this not wrapped in something reusable? + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }, 500) + end + + format :json + + helpers Helpers + helpers ::API::APIHelpers + + mount Builds + mount Commits + mount Runners + mount Projects + mount Forks + mount Triggers + end + end +end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb new file mode 100644 index 00000000000..83ca1e6481c --- /dev/null +++ b/lib/ci/api/builds.rb @@ -0,0 +1,53 @@ +module Ci + module API + # Builds API + class Builds < Grape::API + resource :builds do + # Runs oldest pending build by runner - Runners only + # + # Parameters: + # token (required) - The uniq token of runner + # + # Example Request: + # POST /builds/register + post "register" do + authenticate_runner! + update_runner_last_contact + required_attributes! [:token] + not_found! unless current_runner.active? + + build = Ci::RegisterBuildService.new.execute(current_runner) + + if build + update_runner_info + present build, with: Entities::Build + else + not_found! + end + end + + # Update an existing build - Runners only + # + # Parameters: + # id (required) - The ID of a project + # state (optional) - The state of a build + # trace (optional) - The trace of a build + # Example Request: + # PUT /builds/:id + put ":id" do + authenticate_runner! + update_runner_last_contact + build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) + build.update_attributes(trace: params[:trace]) if params[:trace] + + case params[:state].to_s + when 'success' + build.success + when 'failed' + build.drop + end + end + end + end + end +end diff --git a/lib/ci/api/commits.rb b/lib/ci/api/commits.rb new file mode 100644 index 00000000000..bac463a5909 --- /dev/null +++ b/lib/ci/api/commits.rb @@ -0,0 +1,66 @@ +module Ci + module API + class Commits < Grape::API + resource :commits do + # Get list of commits per project + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # page (optional) + # per_page (optional) - items per request (default is 20) + # + get do + required_attributes! [:project_id, :project_token] + project = Ci::Project.find(params[:project_id]) + authenticate_project_token!(project) + + commits = project.commits.page(params[:page]).per(params[:per_page] || 20) + present commits, with: Entities::CommitWithBuilds + end + + # Create a commit + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # data (required) - GitLab push data + # + # Sample GitLab push data: + # { + # "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", + # "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + # "ref": "refs/heads/master", + # "commits": [ + # { + # "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + # "message": "Update Catalan translation to e38cb41.", + # "timestamp": "2011-12-12T14:27:31+02:00", + # "url": "http://localhost/diaspora/commits/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + # "author": { + # "name": "Jordi Mallach", + # "email": "jordi@softcatala.org", + # } + # }, .... more commits + # ] + # } + # + # Example Request: + # POST /commits + post do + required_attributes! [:project_id, :data, :project_token] + project = Ci::Project.find(params[:project_id]) + authenticate_project_token!(project) + commit = Ci::CreateCommitService.new.execute(project, params[:data]) + + if commit.persisted? + present commit, with: Entities::CommitWithBuilds + else + errors = commit.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + end + end + end +end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb new file mode 100644 index 00000000000..f47bc1236b8 --- /dev/null +++ b/lib/ci/api/entities.rb @@ -0,0 +1,56 @@ +module Ci + module API + module Entities + class Commit < Grape::Entity + expose :id, :ref, :sha, :project_id, :before_sha, :created_at + expose :status, :finished_at, :duration + expose :git_commit_message, :git_author_name, :git_author_email + end + + class CommitWithBuilds < Commit + expose :builds + end + + class Build < Grape::Entity + expose :id, :commands, :ref, :sha, :project_id, :repo_url, + :before_sha, :allow_git_fetch, :project_name + + expose :options do |model| + model.options + end + + expose :timeout do |model| + model.timeout + end + + expose :variables + end + + class Runner < Grape::Entity + expose :id, :token + end + + class Project < Grape::Entity + expose :id, :name, :token, :default_ref, :gitlab_url, :path, + :always_build, :polling_interval, :public, :ssh_url_to_repo, :gitlab_id + + expose :timeout do |model| + model.timeout + end + end + + class RunnerProject < Grape::Entity + expose :id, :project_id, :runner_id + end + + class WebHook < Grape::Entity + expose :id, :project_id, :url + end + + class TriggerRequest < Grape::Entity + expose :id, :variables + expose :commit, using: Commit + end + end + end +end diff --git a/lib/ci/api/forks.rb b/lib/ci/api/forks.rb new file mode 100644 index 00000000000..152883a599f --- /dev/null +++ b/lib/ci/api/forks.rb @@ -0,0 +1,37 @@ +module Ci + module API + class Forks < Grape::API + resource :forks do + # Create a fork + # + # Parameters: + # project_id (required) - The ID of a project + # project_token (requires) - Project token + # private_token(required) - User private token + # data (required) - GitLab project data (name_with_namespace, web_url, default_branch, ssh_url_to_repo) + # + # + # Example Request: + # POST /forks + post do + required_attributes! [:project_id, :data, :project_token, :private_token] + project = Ci::Project.find_by!(gitlab_id: params[:project_id]) + authenticate_project_token!(project) + + fork = Ci::CreateProjectService.new.execute( + current_user, + params[:data], + Ci::RoutesHelper.ci_project_url(":project_id"), + project + ) + + if fork + present fork, with: Entities::Project + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb new file mode 100644 index 00000000000..e602cda81d6 --- /dev/null +++ b/lib/ci/api/helpers.rb @@ -0,0 +1,35 @@ +module Ci + module API + module Helpers + UPDATE_RUNNER_EVERY = 60 + + def authenticate_runners! + forbidden! unless params[:token] == GitlabCi::REGISTRATION_TOKEN + end + + def authenticate_runner! + forbidden! unless current_runner + end + + def authenticate_project_token!(project) + forbidden! unless project.valid_token?(params[:project_token]) + end + + def update_runner_last_contact + if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= UPDATE_RUNNER_EVERY + current_runner.update_attributes(contacted_at: Time.now) + end + end + + def current_runner + @runner ||= Runner.find_by_token(params[:token].to_s) + end + + def update_runner_info + return unless params["info"].present? + info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) + current_runner.update(info) + end + end + end +end diff --git a/lib/ci/api/projects.rb b/lib/ci/api/projects.rb new file mode 100644 index 00000000000..66bcf65e8c4 --- /dev/null +++ b/lib/ci/api/projects.rb @@ -0,0 +1,210 @@ +module Ci + module API + # Projects API + class Projects < Grape::API + before { authenticate! } + + resource :projects do + # Register new webhook for project + # + # Parameters + # project_id (required) - The ID of a project + # web_hook (required) - WebHook URL + # Example Request + # POST /projects/:project_id/webhooks + post ":project_id/webhooks" do + required_attributes! [:web_hook] + + project = Ci::Project.find(params[:project_id]) + + unauthorized! unless can?(current_user, :admin_project, project.gl_project) + + web_hook = project.web_hooks.new({ url: params[:web_hook] }) + + if web_hook.save + present web_hook, with: Entities::WebHook + else + errors = web_hook.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Retrieve all Gitlab CI projects that the user has access to + # + # Example Request: + # GET /projects + get do + gitlab_projects = current_user.authorized_projects + gitlab_projects = filter_projects(gitlab_projects) + gitlab_projects = paginate gitlab_projects + + ids = gitlab_projects.map { |project| project.id } + + projects = Ci::Project.where("gitlab_id IN (?)", ids).load + present projects, with: Entities::Project + end + + # Retrieve all Gitlab CI projects that the user owns + # + # Example Request: + # GET /projects/owned + get "owned" do + gitlab_projects = current_user.owned_projects + gitlab_projects = filter_projects(gitlab_projects) + gitlab_projects = paginate gitlab_projects + + ids = gitlab_projects.map { |project| project.id } + + projects = Ci::Project.where("gitlab_id IN (?)", ids).load + present projects, with: Entities::Project + end + + # Retrieve info for a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # GET /projects/:id + get ":id" do + project = Ci::Project.find(params[:id]) + unauthorized! unless can?(current_user, :read_project, project.gl_project) + + present project, with: Entities::Project + end + + # Create Gitlab CI project using Gitlab project info + # + # Parameters: + # name (required) - The name of the project + # gitlab_id (required) - The gitlab id of the project + # path (required) - The gitlab project path, ex. randx/six + # ssh_url_to_repo (required) - The gitlab ssh url to the repo + # default_ref - The branch to run against (defaults to `master`) + # Example Request: + # POST /projects + post do + required_attributes! [:name, :gitlab_id, :ssh_url_to_repo] + + filtered_params = { + name: params[:name], + gitlab_id: params[:gitlab_id], + # we accept gitlab_url for backward compatibility for a while (added to 7.11) + path: params[:path] || params[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1'), + default_ref: params[:default_ref] || 'master', + ssh_url_to_repo: params[:ssh_url_to_repo] + } + + project = Ci::Project.new(filtered_params) + project.build_missing_services + + if project.save + present project, with: Entities::Project + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Update a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # name - The name of the project + # gitlab_id - The gitlab id of the project + # path - The gitlab project path, ex. randx/six + # ssh_url_to_repo - The gitlab ssh url to the repo + # default_ref - The branch to run against (defaults to `master`) + # Example Request: + # PUT /projects/:id + put ":id" do + project = Ci::Project.find(params[:id]) + + unauthorized! unless can?(current_user, :admin_project, project.gl_project) + + attrs = attributes_for_keys [:name, :gitlab_id, :path, :gitlab_url, :default_ref, :ssh_url_to_repo] + + # we accept gitlab_url for backward compatibility for a while (added to 7.11) + if attrs[:gitlab_url] && !attrs[:path] + attrs[:path] = attrs[:gitlab_url].sub(/.*\/(.*\/.*)$/, '\1') + end + + if project.update_attributes(attrs) + present project, with: Entities::Project + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Remove a Gitlab CI project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id + delete ":id" do + project = Ci::Project.find(params[:id]) + + unauthorized! unless can?(current_user, :admin_project, project.gl_project) + + project.destroy + end + + # Link a Gitlab CI project to a runner + # + # Parameters: + # id (required) - The ID of a CI project + # runner_id (required) - The ID of a runner + # Example Request: + # POST /projects/:id/runners/:runner_id + post ":id/runners/:runner_id" do + project = Ci::Project.find(params[:id]) + runner = Ci::Runner.find(params[:runner_id]) + + unauthorized! unless can?(current_user, :admin_project, project.gl_project) + + options = { + project_id: project.id, + runner_id: runner.id + } + + runner_project = Ci::RunnerProject.new(options) + + if runner_project.save + present runner_project, with: Entities::RunnerProject + else + errors = project.errors.full_messages.join(", ") + render_api_error!(errors, 400) + end + end + + # Remove a Gitlab CI project from a runner + # + # Parameters: + # id (required) - The ID of a CI project + # runner_id (required) - The ID of a runner + # Example Request: + # DELETE /projects/:id/runners/:runner_id + delete ":id/runners/:runner_id" do + project = Ci::Project.find(params[:id]) + runner = Ci::Runner.find(params[:runner_id]) + + unauthorized! unless can?(current_user, :admin_project, project.gl_project) + + options = { + project_id: project.id, + runner_id: runner.id + } + + runner_project = Ci::RunnerProject.find_by(options) + + if runner_project.present? + runner_project.destroy + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb new file mode 100644 index 00000000000..1466fe4356e --- /dev/null +++ b/lib/ci/api/runners.rb @@ -0,0 +1,69 @@ +module Ci + module API + # Runners API + class Runners < Grape::API + resource :runners do + # Get list of all available runners + # + # Example Request: + # GET /runners + get do + authenticate! + runners = Ci::Runner.all + + present runners, with: Entities::Runner + end + + # Delete runner + # Parameters: + # token (required) - The unique token of runner + # + # Example Request: + # GET /runners/delete + delete "delete" do + required_attributes! [:token] + authenticate_runner! + Ci::Runner.find_by_token(params[:token]).destroy + end + + # Register a new runner + # + # Note: This is an "internal" API called when setting up + # runners, so it is authenticated differently. + # + # Parameters: + # token (required) - The unique token of runner + # + # Example Request: + # POST /runners/register + post "register" do + required_attributes! [:token] + + runner = + if params[:token] == GitlabCi::REGISTRATION_TOKEN + # Create shared runner. Requires admin access + Ci::Runner.create( + description: params[:description], + tag_list: params[:tag_list], + is_shared: true + ) + elsif project = Ci::Project.find_by(token: params[:token]) + # Create a specific runner for project. + project.runners.create( + description: params[:description], + tag_list: params[:tag_list] + ) + end + + return forbidden! unless runner + + if runner.id + present runner, with: Entities::Runner + else + not_found! + end + end + end + end + end +end diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb new file mode 100644 index 00000000000..40907d6db54 --- /dev/null +++ b/lib/ci/api/triggers.rb @@ -0,0 +1,49 @@ +module Ci + module API + # Build Trigger API + class Triggers < Grape::API + resource :projects do + # Trigger a GitLab CI project build + # + # Parameters: + # id (required) - The ID of a CI project + # ref (required) - The name of project's branch or tag + # token (required) - The uniq token of trigger + # Example Request: + # POST /projects/:id/ref/:ref/trigger + post ":id/refs/:ref/trigger" do + required_attributes! [:token] + + project = Ci::Project.find(params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token].to_s) + not_found! unless project && trigger + unauthorized! unless trigger.project == project + + # validate variables + variables = params[:variables] + if variables + unless variables.is_a?(Hash) + render_api_error!('variables needs to be a hash', 400) + end + + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + # convert variables from Mash to Hash + variables = variables.to_h + end + + # create request and trigger builds + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + if trigger_request + present trigger_request, with: Entities::TriggerRequest + else + errors = 'No builds created' + render_api_error!(errors, 400) + end + end + end + end + end +end diff --git a/lib/ci/assets/.gitkeep b/lib/ci/assets/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/lib/ci/assets/.gitkeep diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb new file mode 100644 index 00000000000..915a4f526a6 --- /dev/null +++ b/lib/ci/charts.rb @@ -0,0 +1,71 @@ +module Ci + module Charts + class Chart + attr_reader :labels, :total, :success, :project, :build_times + + def initialize(project) + @labels = [] + @total = [] + @success = [] + @build_times = [] + @project = project + + collect + end + + + def push(from, to, format) + @labels << from.strftime(format) + @total << project.builds. + where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). + count(:all) + @success << project.builds. + where("? > #{Ci::Build.table_name}.created_at AND #{Ci::Build.table_name}.created_at > ?", to, from). + success.count(:all) + end + end + + class YearChart < Chart + def collect + 13.times do |i| + start_month = (Date.today.years_ago(1) + i.month).beginning_of_month + end_month = start_month.end_of_month + + push(start_month, end_month, "%d %B %Y") + end + end + end + + class MonthChart < Chart + def collect + 30.times do |i| + start_day = Date.today - 30.days + i.days + end_day = Date.today - 30.days + i.day + 1.day + + push(start_day, end_day, "%d %B") + end + end + end + + class WeekChart < Chart + def collect + 7.times do |i| + start_day = Date.today - 7.days + i.days + end_day = Date.today - 7.days + i.day + 1.day + + push(start_day, end_day, "%d %B") + end + end + end + + class BuildTime < Chart + def collect + commits = project.commits.joins(:builds).where("#{Ci::Build.table_name}.finished_at is NOT NULL AND #{Ci::Build.table_name}.started_at is NOT NULL").last(30) + commits.each do |commit| + @labels << commit.short_sha + @build_times << (commit.duration / 60) + end + end + end + end +end diff --git a/lib/ci/current_settings.rb b/lib/ci/current_settings.rb new file mode 100644 index 00000000000..fd78b024970 --- /dev/null +++ b/lib/ci/current_settings.rb @@ -0,0 +1,22 @@ +module Ci + module CurrentSettings + def current_application_settings + key = :ci_current_application_settings + + RequestStore.store[key] ||= begin + if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('ci_application_settings') + Ci::ApplicationSetting.current || Ci::ApplicationSetting.create_from_defaults + else + fake_application_settings + end + end + end + + def fake_application_settings + OpenStruct.new( + all_broken_builds: Ci::Settings.gitlab_ci['all_broken_builds'], + add_pusher: Ci::Settings.gitlab_ci['add_pusher'], + ) + end + end +end diff --git a/lib/ci/git.rb b/lib/ci/git.rb new file mode 100644 index 00000000000..7acc3f38edb --- /dev/null +++ b/lib/ci/git.rb @@ -0,0 +1,5 @@ +module Ci + module Git + BLANK_SHA = '0' * 40 + end +end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb new file mode 100644 index 00000000000..e625e790df8 --- /dev/null +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -0,0 +1,198 @@ +module Ci + class GitlabCiYamlProcessor + class ValidationError < StandardError;end + + DEFAULT_STAGES = %w(build test deploy) + DEFAULT_STAGE = 'test' + ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables] + ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage] + + attr_reader :before_script, :image, :services, :variables + + def initialize(config) + @config = YAML.load(config) + + unless @config.is_a? Hash + raise ValidationError, "YAML should be a hash" + end + + @config = @config.deep_symbolize_keys + + initial_parsing + + validate! + end + + def builds_for_stage_and_ref(stage, ref, tag = false) + builds.select{|build| build[:stage] == stage && process?(build[:only], build[:except], ref, tag)} + end + + def builds + @jobs.map do |name, job| + build_job(name, job) + end + end + + def stages + @stages || DEFAULT_STAGES + end + + private + + def initial_parsing + @before_script = @config[:before_script] || [] + @image = @config[:image] + @services = @config[:services] + @stages = @config[:stages] || @config[:types] + @variables = @config[:variables] || {} + @config.except!(*ALLOWED_YAML_KEYS) + + # anything that doesn't have script is considered as unknown + @config.each do |name, param| + raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash) && param.has_key?(:script) + end + + unless @config.values.any?{|job| job.is_a?(Hash)} + raise ValidationError, "Please define at least one job" + end + + @jobs = {} + @config.each do |key, job| + stage = job[:stage] || job[:type] || DEFAULT_STAGE + @jobs[key] = { stage: stage }.merge(job) + end + end + + def process?(only_params, except_params, ref, tag) + return true if only_params.nil? && except_params.nil? + + if only_params + return true if tag && only_params.include?("tags") + return true if !tag && only_params.include?("branches") + + only_params.find do |pattern| + match_ref?(pattern, ref) + end + else + return false if tag && except_params.include?("tags") + return false if !tag && except_params.include?("branches") + + except_params.each do |pattern| + return false if match_ref?(pattern, ref) + end + end + end + + def build_job(name, job) + { + stage: job[:stage], + script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}", + tags: job[:tags] || [], + name: name, + only: job[:only], + except: job[:except], + allow_failure: job[:allow_failure] || false, + options: { + image: job[:image] || @image, + services: job[:services] || @services + }.compact + } + end + + def match_ref?(pattern, ref) + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ ref + else + pattern == ref + end + end + + def normalize_script(script) + if script.is_a? Array + script.join("\n") + else + script + end + end + + def validate! + unless validate_array_of_strings(@before_script) + raise ValidationError, "before_script should be an array of strings" + end + + unless @image.nil? || @image.is_a?(String) + raise ValidationError, "image should be a string" + end + + unless @services.nil? || validate_array_of_strings(@services) + raise ValidationError, "services should be an array of strings" + end + + unless @stages.nil? || validate_array_of_strings(@stages) + raise ValidationError, "stages should be an array of strings" + end + + unless @variables.nil? || validate_variables(@variables) + raise ValidationError, "variables should be a map of key-valued strings" + end + + @jobs.each do |name, job| + validate_job!("#{name} job", job) + end + + true + end + + def validate_job!(name, job) + job.keys.each do |key| + unless ALLOWED_JOB_KEYS.include? key + raise ValidationError, "#{name}: unknown parameter #{key}" + end + end + + if !job[:script].is_a?(String) && !validate_array_of_strings(job[:script]) + raise ValidationError, "#{name}: script should be a string or an array of a strings" + end + + if job[:stage] + unless job[:stage].is_a?(String) && job[:stage].in?(stages) + raise ValidationError, "#{name}: stage parameter should be #{stages.join(", ")}" + end + end + + if job[:image] && !job[:image].is_a?(String) + raise ValidationError, "#{name}: image should be a string" + end + + if job[:services] && !validate_array_of_strings(job[:services]) + raise ValidationError, "#{name}: services should be an array of strings" + end + + if job[:tags] && !validate_array_of_strings(job[:tags]) + raise ValidationError, "#{name}: tags parameter should be an array of strings" + end + + if job[:only] && !validate_array_of_strings(job[:only]) + raise ValidationError, "#{name}: only parameter should be an array of strings" + end + + if job[:except] && !validate_array_of_strings(job[:except]) + raise ValidationError, "#{name}: except parameter should be an array of strings" + end + + if job[:allow_failure] && !job[:allow_failure].in?([true, false]) + raise ValidationError, "#{name}: allow_failure parameter should be an boolean" + end + end + + private + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? {|tag| tag.is_a?(String)} + end + + def validate_variables(variables) + variables.is_a?(Hash) && variables.all? {|key, value| key.is_a?(Symbol) && value.is_a?(String)} + end + end +end diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb new file mode 100644 index 00000000000..74f592dcaea --- /dev/null +++ b/lib/ci/migrate/database.rb @@ -0,0 +1,67 @@ +require 'yaml' + +module Ci + module Migrate + class Database + attr_reader :config + + def initialize + @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] + end + + def restore(ci_dump) + puts 'Deleting all CI related data ... ' + truncate_ci_tables + + puts 'Restoring CI data ... ' + case config["adapter"] + when /^mysql/ then + print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + system('mysql', *mysql_args, config['database'], in: ci_dump) + when "postgresql" then + puts "Restoring PostgreSQL database #{config['database']} ... " + pg_env + system('psql', config['database'], '-f', ci_dump) + end + end + + protected + + def truncate_ci_tables + c = ActiveRecord::Base.connection + c.tables.select { |t| t.start_with?('ci_') }.each do |table| + puts "Deleting data from #{table}..." + c.execute("DELETE FROM #{table}") + end + end + + def mysql_args + args = { + 'host' => '--host', + 'port' => '--port', + 'socket' => '--socket', + 'username' => '--user', + 'encoding' => '--default-character-set' + } + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact + end + + def pg_env + ENV['PGUSER'] = config["username"] if config["username"] + ENV['PGHOST'] = config["host"] if config["host"] + ENV['PGPORT'] = config["port"].to_s if config["port"] + ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + end + + def report_success(success) + if success + puts '[DONE]'.green + else + puts '[FAILED]'.red + end + end + end + end +end diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb new file mode 100644 index 00000000000..125a535e9a9 --- /dev/null +++ b/lib/ci/migrate/tags.rb @@ -0,0 +1,49 @@ +require 'yaml' + +module Ci + module Migrate + class Tags + def restore + puts 'Migrating tags for Runners... ' + list_objects('Runner').each do |id| + putc '.' + runner = Ci::Runner.find_by_id(id) + if runner + tags = list_tags('Runner', id) + runner.update_attributes(tag_list: tags) + end + end + puts '' + + puts 'Migrating tags for Builds... ' + list_objects('Build').each do |id| + putc '.' + build = Ci::Build.find_by_id(id) + if build + tags = list_tags('Build', id) + build.update_attributes(tag_list: tags) + end + end + puts '' + end + + protected + + def list_objects(type) + ids = ActiveRecord::Base.connection.select_all( + "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}" + ) + ids.map { |id| id['taggable_id'] } + end + + def list_tags(type, id) + tags = ActiveRecord::Base.connection.select_all( + 'select ci_tags.name from ci_tags ' + + 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + + "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'" + ) + tags.map { |tag| tag['name'] } + end + end + end +end diff --git a/lib/ci/model.rb b/lib/ci/model.rb new file mode 100644 index 00000000000..c42a0ad36db --- /dev/null +++ b/lib/ci/model.rb @@ -0,0 +1,11 @@ +module Ci + module Model + def table_name_prefix + "ci_" + end + + def model_name + @model_name ||= ActiveModel::Name.new(self, nil, self.name.split("::").last) + end + end +end diff --git a/lib/ci/scheduler.rb b/lib/ci/scheduler.rb new file mode 100644 index 00000000000..ee0958f4be1 --- /dev/null +++ b/lib/ci/scheduler.rb @@ -0,0 +1,16 @@ +module Ci + class Scheduler + def perform + projects = Ci::Project.where(always_build: true).all + projects.each do |project| + last_commit = project.commits.last + next unless last_commit && last_commit.last_build + + interval = project.polling_interval + if (last_commit.last_build.created_at + interval.hours) < Time.now + last_commit.retry + end + end + end + end +end diff --git a/lib/ci/static_model.rb b/lib/ci/static_model.rb new file mode 100644 index 00000000000..bb2bdbed495 --- /dev/null +++ b/lib/ci/static_model.rb @@ -0,0 +1,49 @@ +# Provides an ActiveRecord-like interface to a model whose data is not persisted to a database. +module Ci + module StaticModel + extend ActiveSupport::Concern + + module ClassMethods + # Used by ActiveRecord's polymorphic association to set object_id + def primary_key + 'id' + end + + # Used by ActiveRecord's polymorphic association to set object_type + def base_class + self + end + end + + # Used by AR for fetching attributes + # + # Pass it along if we respond to it. + def [](key) + send(key) if respond_to?(key) + end + + def to_param + id + end + + def new_record? + false + end + + def persisted? + false + end + + def destroyed? + false + end + + def ==(other) + if other.is_a? ::Ci::StaticModel + id == other.id + else + super + end + end + end +end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb new file mode 100644 index 00000000000..2a87c91db5e --- /dev/null +++ b/lib/ci/version_info.rb @@ -0,0 +1,52 @@ +class VersionInfo + include Comparable + + attr_reader :major, :minor, :patch + + def self.parse(str) + if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) + VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) + else + VersionInfo.new + end + end + + def initialize(major = 0, minor = 0, patch = 0) + @major = major + @minor = minor + @patch = patch + end + + def <=>(other) + return unless other.is_a? VersionInfo + return unless valid? && other.valid? + + if other.major < @major + 1 + elsif @major < other.major + -1 + elsif other.minor < @minor + 1 + elsif @minor < other.minor + -1 + elsif other.patch < @patch + 1 + elsif @patch < other.patch + -1 + else + 0 + end + end + + def to_s + if valid? + "%d.%d.%d" % [@major, @minor, @patch] + else + "Unknown" + end + end + + def valid? + @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 + end +end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index 12292f614e9..0353b3b7ed3 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -1,6 +1,14 @@ require_relative 'shell_env' module Grack + class AuthSpawner + def self.call(env) + # Avoid issues with instance variables in Grack::Auth persisting across + # requests by creating a new instance for each request. + Auth.new({}).call(env) + end + end + class Auth < Rack::Auth::Basic attr_accessor :user, :project, :env @@ -10,7 +18,7 @@ module Grack @request = Rack::Request.new(env) @auth = Request.new(env) - @gitlab_ci = false + @ci = false # Need this patch due to the rails mount # Need this if under RELATIVE_URL_ROOT @@ -26,13 +34,9 @@ module Grack auth! if project && authorized_request? - if ENV['GITLAB_GRACK_AUTH_ONLY'] == '1' - # Tell gitlab-git-http-server the request is OK, and what the GL_ID is - render_grack_auth_ok - else - @app.call(env) - end - elsif @user.nil? && !@gitlab_ci + # Tell gitlab-git-http-server the request is OK, and what the GL_ID is + render_grack_auth_ok + elsif @user.nil? && !@ci unauthorized else render_not_found @@ -51,8 +55,8 @@ module Grack # Allow authentication for GitLab CI service # if valid token passed - if gitlab_ci_request?(login, password) - @gitlab_ci = true + if ci_request?(login, password) + @ci = true return end @@ -64,12 +68,17 @@ module Grack end end - def gitlab_ci_request?(login, password) - if login == "gitlab-ci-token" && project && project.gitlab_ci? - token = project.gitlab_ci_service.token + def ci_request?(login, password) + matched_login = /(?<s>^[a-zA-Z]*-ci)-token$/.match(login) + + if project && matched_login.present? && git_cmd == 'git-upload-pack' + underscored_service = matched_login['s'].underscore - if token.present? && token == password && git_cmd == 'git-upload-pack' - return true + if Service.available_services_names.include?(underscored_service) + service_method = "#{underscored_service}_service" + service = project.send(service_method) + + return service && service.activated? && service.valid_token?(password) end end @@ -128,11 +137,13 @@ module Grack end def authorized_request? - return true if @gitlab_ci + return true if @ci case git_cmd when *Gitlab::GitAccess::DOWNLOAD_COMMANDS - if user + if !Gitlab.config.gitlab_shell.upload_pack + false + elsif user Gitlab::GitAccess.new(user, project).download_access_check.allowed? elsif project.public? # Allow clone/fetch for public projects @@ -141,7 +152,9 @@ module Grack false end when *Gitlab::GitAccess::PUSH_COMMANDS - if user + if !Gitlab.config.gitlab_shell.receive_pack + false + elsif user # Skip user authorization on upload request. # It will be done by the pre-receive hook in the repository. true diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb index aec44b8c87b..d88a6eaac6b 100644 --- a/lib/gitlab/bitbucket_import/client.rb +++ b/lib/gitlab/bitbucket_import/client.rb @@ -52,11 +52,26 @@ module Gitlab end def issues(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues").body) + all_issues = [] + offset = 0 + per_page = 50 # Maximum number allowed by Bitbucket + index = 0 + + begin + issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) + # Find out how many total issues are present + total = issues["count"] if index == 0 + all_issues.concat(issues["issues"]) + offset += issues["issues"].count + index += 1 + end while all_issues.count < total + + all_issues end def issue_comments(project_identifier, issue_id) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) + comments.sort_by { |comment| comment["utc_created_on"] } end def project(project_identifier) @@ -100,6 +115,10 @@ module Gitlab response end + def issue_api_endpoint(project_identifier, per_page, offset) + "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"} end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 42c93707caa..2355b3c6ddc 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -5,7 +5,10 @@ module Gitlab def initialize(project) @project = project - @client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret) + import_data = project.import_data.try(:data) + bb_session = import_data["bb_session"] if import_data + @client = Client.new(bb_session["bitbucket_access_token"], + bb_session["bitbucket_access_token_secret"]) @formatter = Gitlab::ImportFormatter.new end @@ -16,37 +19,57 @@ module Gitlab #Issues && Comments issues = client.issues(project_identifier) - - issues["issues"].each do |issue| - body = @formatter.author_line(issue["reported_by"]["username"], issue["content"]) - + + issues.each do |issue| + body = '' + reporter = nil + author = 'Anonymous' + + if issue["reported_by"] && issue["reported_by"]["username"] + reporter = issue["reported_by"]["username"] + author = reporter + end + + body = @formatter.author_line(author) + body += issue["content"] + comments = client.issue_comments(project_identifier, issue["local_id"]) - + if comments.any? body += @formatter.comments_header end comments.each do |comment| - body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"]) + author = 'Anonymous' + + if comment["author_info"] && comment["author_info"]["username"] + author = comment["author_info"]["username"] + end + + body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) end project.issues.create!( - description: body, + description: body, title: issue["title"], state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, issue["reported_by"]["username"]) + author_id: gl_user_id(project, reporter) ) end - + true end private def gl_user_id(project, bitbucket_id) - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - end + if bitbucket_id + user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) + (user && user.id) || project.creator_id + else + project.creator_id + end + end end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb index 9931aa7e029..0b63f025d0a 100644 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ b/lib/gitlab/bitbucket_import/key_adder.rb @@ -3,14 +3,15 @@ module Gitlab class KeyAdder attr_reader :repo, :current_user, :client - def initialize(repo, current_user) + def initialize(repo, current_user, access_params) @repo, @current_user = repo, current_user - @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret) + @client = Client.new(access_params[:bitbucket_access_token], + access_params[:bitbucket_access_token_secret]) end def execute return false unless BitbucketImport.public_key.present? - + project_identifier = "#{repo["owner"]}/#{repo["slug"]}" client.add_deploy_key(project_identifier, BitbucketImport.public_key) diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb index 1a24a86fc37..f4dd393ad29 100644 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ b/lib/gitlab/bitbucket_import/key_deleter.rb @@ -6,12 +6,15 @@ module Gitlab def initialize(project) @project = project @current_user = project.creator - @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret) + import_data = project.import_data.try(:data) + bb_session = import_data["bb_session"] if import_data + @client = Client.new(bb_session["bitbucket_access_token"], + bb_session["bitbucket_access_token_secret"]) end def execute return false unless BitbucketImport.public_key.present? - + client.delete_deploy_key(project.import_source, BitbucketImport.public_key) true diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index 54420e62c90..35e34d033e0 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,16 +1,17 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user + attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user) + def initialize(repo, namespace, current_user, session_data) @repo = repo @namespace = namespace @current_user = current_user + @session_data = session_data end def execute - ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new(current_user, name: repo["name"], path: repo["slug"], description: repo["description"], @@ -18,8 +19,11 @@ module Gitlab visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "bitbucket", import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git" + import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", ).execute + + project.create_import_data(data: { "bb_session" => session_data } ) + project end end end diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb new file mode 100644 index 00000000000..9c4664df903 --- /dev/null +++ b/lib/gitlab/color_schemes.rb @@ -0,0 +1,67 @@ +module Gitlab + # Module containing GitLab's syntax color scheme definitions and helper + # methods for accessing them. + module ColorSchemes + # Struct class representing a single Scheme + Scheme = Struct.new(:id, :name, :css_class) + + SCHEMES = [ + Scheme.new(1, 'White', 'white'), + Scheme.new(2, 'Dark', 'dark'), + Scheme.new(3, 'Solarized Light', 'solarized-light'), + Scheme.new(4, 'Solarized Dark', 'solarized-dark'), + Scheme.new(5, 'Monokai', 'monokai') + ].freeze + + # Convenience method to get a space-separated String of all the color scheme + # classes that might be applied to a code block. + # + # Returns a String + def self.body_classes + SCHEMES.collect(&:css_class).uniq.join(' ') + end + + # Get a Scheme by its ID + # + # If the ID is invalid, returns the default Scheme. + # + # id - Integer ID + # + # Returns a Scheme + def self.by_id(id) + SCHEMES.detect { |s| s.id == id } || default + end + + # Returns the number of defined Schemes + def self.count + SCHEMES.size + end + + # Get the default Scheme + # + # Returns a Scheme + def self.default + by_id(1) + end + + # Iterate through each Scheme + # + # Yields the Scheme object + def self.each(&block) + SCHEMES.each(&block) + end + + # Get the Scheme for the specified user, or the default + # + # user - User record + # + # Returns a Scheme + def self.for_user(user) + if user + by_id(user.color_scheme_id) + else + default + end + end + end +end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 1a2a50a14d0..0ea1b6a2f6f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -4,7 +4,7 @@ module Gitlab key = :current_application_settings RequestStore.store[key] ||= begin - if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('application_settings') + if connect_to_db? ApplicationSetting.current || ApplicationSetting.create_from_defaults else fake_application_settings @@ -26,5 +26,17 @@ module Gitlab import_sources: Settings.gitlab['import_sources'] ) end + + private + + def connect_to_db? + use_db = if ENV['USE_DB'] == "false" + false + else + true + end + + use_db && ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings') + end end end diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb new file mode 100644 index 00000000000..431d50882fd --- /dev/null +++ b/lib/gitlab/fogbugz_import/client.rb @@ -0,0 +1,56 @@ +require 'fogbugz' + +module Gitlab + module FogbugzImport + class Client + attr_reader :api + + def initialize(options = {}) + if options[:uri] && options[:token] + @api = ::Fogbugz::Interface.new(options) + elsif options[:uri] && options[:email] && options[:password] + @api = ::Fogbugz::Interface.new(options) + @api.authenticate + @api + end + end + + def get_token + @api.token + end + + def valid? + !get_token.blank? + end + + def user_map + users = {} + res = @api.command(:listPeople) + res['people']['person'].each do |user| + users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] } + end + users + end + + def repos + res = @api.command(:listProjects) + @repos ||= res['projects']['project'].map { |proj| FogbugzImport::Repository.new(proj) } + end + + def repo(id) + repos.find { |r| r.id.to_s == id.to_s } + end + + def cases(project_id) + project_name = repo(project_id).name + res = @api.command(:search, q: "project:'#{project_name}'", cols: 'ixPersonAssignedTo,ixPersonOpenedBy,ixPersonClosedBy,sStatus,sPriority,sCategory,fOpen,sTitle,sLatestTextSummary,dtOpened,dtClosed,dtResolved,dtLastUpdated,events') + return [] unless res['cases']['count'].to_i > 0 + res['cases']['case'] + end + + def categories + @api.command(:listCategories) + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb new file mode 100644 index 00000000000..61e08b23543 --- /dev/null +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -0,0 +1,298 @@ +module Gitlab + module FogbugzImport + class Importer + attr_reader :project, :repo + + def initialize(project) + @project = project + + import_data = project.import_data.try(:data) + repo_data = import_data['repo'] if import_data + @repo = FogbugzImport::Repository.new(repo_data) + + @known_labels = Set.new + end + + def execute + return true unless repo.valid? + + data = project.import_data.try(:data) + + client = Gitlab::FogbugzImport::Client.new(token: data['fb_session']['token'], uri: data['fb_session']['uri']) + + @cases = client.cases(@repo.id.to_i) + @categories = client.categories + + import_cases + + true + end + + private + + def user_map + @user_map ||= begin + user_map = Hash.new + import_data = project.import_data.try(:data) + stored_user_map = import_data['user_map'] if import_data + user_map.update(stored_user_map) if stored_user_map + + user_map + end + end + + def import_labels + @categories['categories']['category'].each do |label| + create_label(label['sCategory']) + @known_labels << name + end + end + + def nice_label_color(name) + case name + when 'Blocker' + '#ff0000' + when 'Crash' + '#ffcfcf' + when 'Major' + '#deffcf' + when 'Minor' + '#cfe9ff' + when 'Bug' + '#d9534f' + when 'Feature' + '#44ad8e' + when 'Technical Task' + '#4b6dd0' + else + '#e2e2e2' + end + end + + def create_label(name) + color = nice_label_color(name) + Label.create!(project_id: project.id, title: name, color: color) + end + + def user_info(person_id) + user_hash = user_map[person_id.to_s] + + user_name = '' + gitlab_id = nil + + unless user_hash.nil? + user_name = user_hash['name'] + if user = User.find_by(id: user_hash['gitlab_user']) + user_name = "@#{user.username}" + gitlab_id = user.id + end + end + + { name: user_name, gitlab_id: gitlab_id } + end + + def import_cases + return unless @cases + + while bug = @cases.shift + author = user_info(bug['ixPersonOpenedBy'])[:name] + date = DateTime.parse(bug['dtOpened']) + + comments = bug['events']['event'] + + content = format_content(opened_content(comments)) + body = format_issue_body(author, date, content) + + labels = [] + [bug['sCategory'], bug['sPriority']].each do |label| + unless label.blank? + labels << label + unless @known_labels.include?(label) + create_label(label) + @known_labels << label + end + end + end + + assignee_id = user_info(bug['ixPersonAssignedTo'])[:gitlab_id] + author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id + + issue = Issue.create!( + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_id: assignee_id, + state: bug['fOpen'] == 'true' ? 'opened' : 'closed' + ) + issue.add_labels_by_names(labels) + + if issue.iid != bug['ixBug'] + issue.update_attribute(:iid, bug['ixBug']) + end + + import_issue_comments(issue, comments) + + issue.update_attribute(:created_at, date) + + last_update = DateTime.parse(bug['dtLastUpdated']) + issue.update_attribute(:updated_at, last_update) + end + end + + def opened_content(comments) + while comment = comments.shift + if comment['sVerb'] == 'Opened' + return comment['s'] + end + end + '' + end + + def import_issue_comments(issue, comments) + Note.transaction do + while comment = comments.shift + verb = comment['sVerb'] + + next if verb == 'Opened' || verb === 'Closed' + + content = format_content(comment['s']) + attachments = format_attachments(comment['rgAttachments']) + updates = format_updates(comment) + + next if content.blank? && attachments.empty? && updates.empty? + + author = user_info(comment['ixPerson'])[:name] + author_id = user_info(comment['ixPerson'])[:gitlab_id] || project.creator_id + date = DateTime.parse(comment['dt']) + + body = format_issue_comment_body( + comment['ixBugEvent'], + author, + date, + content, + attachments, + updates + ) + + note = Note.create!( + project_id: project.id, + noteable_type: "Issue", + noteable_id: issue.id, + author_id: author_id, + note: body + ) + + note.update_attribute(:created_at, date) + note.update_attribute(:updated_at, date) + end + end + end + + def linkify_issues(s) + s = s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') + s = s.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') + s + end + + def escape_for_markdown(s) + s = s.gsub(/^#/, "\\#") + s = s.gsub(/^-/, "\\-") + s = s.gsub("`", "\\~") + s = s.gsub("\r", "") + s = s.gsub("\n", " \n") + s + end + + def format_content(raw_content) + return raw_content if raw_content.nil? + linkify_issues(escape_for_markdown(raw_content)) + end + + def format_attachments(raw_attachments) + return [] unless raw_attachments + + attachments = case raw_attachments['attachment'] + when Array + raw_attachments['attachment'] + when Hash + [raw_attachments['attachment']] + else + [] + end + + attachments.map! { |a| format_attachment(a) } + attachments.compact + end + + def format_attachment(attachment) + link = build_attachment_url(attachment['sURL']) + + res = ::Projects::DownloadService.new(project, link).execute + + return nil if res.nil? + + text = "[#{res['alt']}](#{res['url']})" + text = "!#{text}" if res['is_image'] + text + end + + def build_attachment_url(rel_url) + data = project.import_data.try(:data) + uri = data['fb_session']['uri'] + token = data['fb_session']['token'] + "#{uri}/#{rel_url}&token=#{token}" + end + + def format_updates(comment) + updates = [] + + if comment['sChanges'] + updates << "*Changes: #{linkify_issues(comment['sChanges'].chomp)}*" + end + + if comment['evtDescription'] + updates << "*#{comment['evtDescription']}*" + end + + updates + end + + def format_issue_body(author, date, content) + body = [] + body << "*By #{author} on #{date} (imported from FogBugz)*" + body << '---' + + if content.blank? + content = '*(No description has been entered for this issue)*' + end + body << content + + body.join("\n\n") + end + + def format_issue_comment_body(id, author, date, content, attachments, updates) + body = [] + body << "*By #{author} on #{date} (imported from FogBugz)*" + body << '---' + + if content.blank? + content = "*(No comment has been entered for this change)*" + end + body << content + + if updates.any? + body << '---' + body += updates + end + + if attachments.any? + body << '---' + body += attachments + end + + body.join("\n\n") + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb new file mode 100644 index 00000000000..f02ea43910f --- /dev/null +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -0,0 +1,38 @@ +module Gitlab + module FogbugzImport + class ProjectCreator + attr_reader :repo, :fb_session, :namespace, :current_user, :user_map + + def initialize(repo, fb_session, namespace, current_user, user_map = nil) + @repo = repo + @fb_session = fb_session + @namespace = namespace + @current_user = current_user + @user_map = user_map + end + + def execute + project = ::Projects::CreateService.new(current_user, + name: repo.safe_name, + path: repo.path, + namespace: namespace, + creator: current_user, + visibility_level: Gitlab::VisibilityLevel::INTERNAL, + import_type: 'fogbugz', + import_source: repo.name, + import_url: Project::UNKNOWN_IMPORT_URL + ).execute + + import_data = project.create_import_data( + data: { + 'repo' => repo.raw_data, + 'user_map' => user_map, + 'fb_session' => fb_session + } + ) + + project + end + end + end +end diff --git a/lib/gitlab/fogbugz_import/repository.rb b/lib/gitlab/fogbugz_import/repository.rb new file mode 100644 index 00000000000..d1dc63db2b2 --- /dev/null +++ b/lib/gitlab/fogbugz_import/repository.rb @@ -0,0 +1,31 @@ +module Gitlab + module FogbugzImport + class Repository + attr_accessor :raw_data + + def initialize(raw_data) + @raw_data = raw_data + end + + def valid? + raw_data.is_a?(Hash) + end + + def id + raw_data['ixProject'] + end + + def name + raw_data['sProject'] + end + + def safe_name + name.gsub(/[^\s\w.-]/, '') + end + + def path + safe_name.gsub(/[\s]/, '_') + end + end + end +end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 98039a76dcd..bd7340a80f1 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -5,7 +5,9 @@ module Gitlab def initialize(project) @project = project - @client = Client.new(project.creator.github_access_token) + import_data = project.import_data.try(:data) + github_session = import_data["github_session"] if import_data + @client = Client.new(github_session["github_access_token"]) @formatter = Gitlab::ImportFormatter.new end @@ -16,7 +18,8 @@ module Gitlab direction: :asc).each do |issue| if issue.pull_request.nil? - body = @formatter.author_line(issue.user.login, issue.body) + body = @formatter.author_line(issue.user.login) + body += issue.body if issue.comments > 0 body += @formatter.comments_header diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 2723eec933e..8c27ebd1ce8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,16 +1,18 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user + attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user) + def initialize(repo, namespace, current_user, session_data) @repo = repo @namespace = namespace @current_user = current_user + @session_data = session_data end def execute - ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new( + current_user, name: repo.name, path: repo.name, description: repo.description, @@ -18,8 +20,11 @@ module Gitlab visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@") + import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@") ).execute + + project.create_import_data(data: { "github_session" => session_data } ) + project end end end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index c5304a0699b..e24b94d6159 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -5,7 +5,9 @@ module Gitlab def initialize(project) @project = project - @client = Client.new(project.creator.gitlab_access_token) + import_data = project.import_data.try(:data) + gitlab_session = import_data["gitlab_session"] if import_data + @client = Client.new(gitlab_session["gitlab_access_token"]) @formatter = Gitlab::ImportFormatter.new end @@ -14,12 +16,13 @@ module Gitlab #Issues && Comments issues = client.issues(project_identifier) - + issues.each do |issue| - body = @formatter.author_line(issue["author"]["name"], issue["description"]) - + body = @formatter.author_line(issue["author"]["name"]) + body += issue["description"] + comments = client.issue_comments(project_identifier, issue["id"]) - + if comments.any? body += @formatter.comments_header end @@ -29,13 +32,13 @@ module Gitlab end project.issues.create!( - description: body, + description: body, title: issue["title"], state: issue["state"], author_id: gl_user_id(project, issue["author"]["id"]) ) end - + true end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index f0d7141bf56..d9452de6a50 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -1,16 +1,17 @@ module Gitlab module GitlabImport class ProjectCreator - attr_reader :repo, :namespace, :current_user + attr_reader :repo, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user) + def initialize(repo, namespace, current_user, session_data) @repo = repo @namespace = namespace @current_user = current_user + @session_data = session_data end def execute - ::Projects::CreateService.new(current_user, + project = ::Projects::CreateService.new(current_user, name: repo["name"], path: repo["path"], description: repo["description"], @@ -18,8 +19,11 @@ module Gitlab visibility_level: repo["visibility_level"], import_type: "gitlab", import_source: repo["path_with_namespace"], - import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{current_user.gitlab_access_token}@") + import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute + + project.create_import_data(data: { "gitlab_session" => session_data } ) + project end end end diff --git a/lib/gitlab/import_formatter.rb b/lib/gitlab/import_formatter.rb index 72e041a90b1..3e54456e936 100644 --- a/lib/gitlab/import_formatter.rb +++ b/lib/gitlab/import_formatter.rb @@ -8,8 +8,8 @@ module Gitlab "\n\n\n**Imported comments:**\n" end - def author_line(author, body) - "*Created by: #{author}*\n\n#{body}" + def author_line(author) + "*Created by: #{author}*\n\n" end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 991b70aab6a..ccfdfbe73e8 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -19,6 +19,7 @@ module Gitlab 'GitLab.com' => 'gitlab', 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', + 'FogBugz' => 'fogbugz', 'Any repo by URL' => 'git', } end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb new file mode 100644 index 00000000000..55deeeacd90 --- /dev/null +++ b/lib/gitlab/ldap/auth_hash.rb @@ -0,0 +1,35 @@ +# Class to parse and transform the info provided by omniauth +# +module Gitlab + module LDAP + class AuthHash < Gitlab::OAuth::AuthHash + private + + def get_info(key) + attributes = ldap_config.attributes[key] + return super unless attributes + + attributes = Array(attributes) + + value = nil + attributes.each do |attribute| + value = get_raw(attribute) + break if value.present? + end + + return super unless value + + Gitlab::Utils.force_utf8(value) + value + end + + def get_raw(key) + auth_hash.extra[:raw_info][key] + end + + def ldap_config + @ldap_config ||= Gitlab::LDAP::Config.new(self.provider) + end + end + end +end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index d2ffa2e1fe8..101a3285f4b 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -84,6 +84,10 @@ module Gitlab options['block_auto_created_users'] end + def attributes + options['attributes'] + end + protected def base_config Gitlab.config.ldap diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index f7f3ba9ad7d..cb66fd500fe 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -44,9 +44,14 @@ module Gitlab gl_user.skip_reconfirmation! gl_user.email = auth_hash.email - # Build new identity only if we dont have have same one - gl_user.identities.find_or_initialize_by(provider: auth_hash.provider, - extern_uid: auth_hash.uid) + # find_or_initialize_by doesn't update `gl_user.identities`, and isn't autosaved. + identity = gl_user.identities.find { |identity| identity.provider == auth_hash.provider } + identity ||= gl_user.identities.build(provider: auth_hash.provider) + + # For a new user set extern_uid to the LDAP DN + # For an existing user with matching email but changed DN, update the DN. + # For an existing user with no change in DN, this line changes nothing. + identity.extern_uid = auth_hash.uid gl_user end @@ -66,6 +71,10 @@ module Gitlab def ldap_config Gitlab::LDAP::Config.new(auth_hash.provider) end + + def auth_hash=(auth_hash) + @auth_hash = Gitlab::LDAP::AuthHash.new(auth_hash) + end end end end diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb index 9f6e19a09fd..ae5f2544691 100644 --- a/lib/gitlab/markdown.rb +++ b/lib/gitlab/markdown.rb @@ -5,6 +5,32 @@ module Gitlab # # See the files in `lib/gitlab/markdown/` for specific processing information. module Markdown + # Convert a Markdown String into an HTML-safe String of HTML + # + # markdown - Markdown String + # context - Hash of context options passed to our HTML Pipeline + # + # Returns an HTML-safe String + def self.render(markdown, context = {}) + html = renderer.render(markdown) + html = gfm(html, context) + + html.html_safe + end + + # Convert a Markdown String into HTML without going through the HTML + # Pipeline. + # + # Note that because the pipeline is skipped, SanitizationFilter is as well. + # Do not output the result of this method to the user. + # + # markdown - Markdown String + # + # Returns a String + def self.render_without_gfm(markdown) + renderer.render(markdown) + end + # Provide autoload paths for filters to prevent a circular dependency error autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter' autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter' @@ -18,6 +44,7 @@ module Gitlab autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter' autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter' autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter' + autoload :SyntaxHighlightFilter, 'gitlab/markdown/syntax_highlight_filter' autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter' autoload :TaskListFilter, 'gitlab/markdown/task_list_filter' autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter' @@ -28,8 +55,7 @@ module Gitlab # options - A Hash of options used to customize output (default: {}): # :xhtml - output XHTML instead of HTML # :reference_only_path - Use relative path for reference links - # html_options - extra options for the reference links as given to link_to - def gfm(text, options = {}, html_options = {}) + def self.gfm(text, options = {}) return text if text.nil? # Duplicate the string so we don't alter the original, then call to_str @@ -40,8 +66,8 @@ module Gitlab options.reverse_merge!( xhtml: false, reference_only_path: true, - project: @project, - current_user: current_user + project: options[:project], + current_user: options[:current_user] ) @pipeline ||= HTML::Pipeline.new(filters) @@ -51,7 +77,7 @@ module Gitlab pipeline: options[:pipeline], # EmojiFilter - asset_root: Gitlab.config.gitlab.url, + asset_root: Gitlab.config.gitlab.base_url, asset_host: Gitlab::Application.config.asset_host, # TableOfContentsFilter @@ -61,12 +87,11 @@ module Gitlab current_user: options[:current_user], only_path: options[:reference_only_path], project: options[:project], - reference_class: html_options[:class], # RelativeLinkFilter - ref: @ref, - requested_path: @path, - project_wiki: @project_wiki + ref: options[:ref], + requested_path: options[:path], + project_wiki: options[:project_wiki] } result = @pipeline.call(text, context) @@ -83,14 +108,36 @@ module Gitlab private + def self.renderer + @markdown ||= begin + renderer = Redcarpet::Render::HTML.new + Redcarpet::Markdown.new(renderer, redcarpet_options) + end + end + + def self.redcarpet_options + # https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use + @redcarpet_options ||= { + fenced_code_blocks: true, + footnotes: true, + lax_spacing: true, + no_intra_emphasis: true, + space_after_headers: true, + strikethrough: true, + superscript: true, + tables: true + }.freeze + end + # Filters used in our pipeline # # SanitizationFilter should come first so that all generated reference HTML # goes through untouched. # # See https://github.com/jch/html-pipeline#filters for more filters. - def filters + def self.filters [ + Gitlab::Markdown::SyntaxHighlightFilter, Gitlab::Markdown::SanitizationFilter, Gitlab::Markdown::RelativeLinkFilter, diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/autolink_filter.rb index 541f1d88ffc..c37c3bc55bf 100644 --- a/lib/gitlab/markdown/autolink_filter.rb +++ b/lib/gitlab/markdown/autolink_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'uri' diff --git a/lib/gitlab/markdown/commit_range_reference_filter.rb b/lib/gitlab/markdown/commit_range_reference_filter.rb index a9f1ee9c161..bb496135d92 100644 --- a/lib/gitlab/markdown/commit_range_reference_filter.rb +++ b/lib/gitlab/markdown/commit_range_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces commit range references with links. @@ -71,7 +73,7 @@ module Gitlab end def url_for_commit_range(project, range) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/gitlab/markdown/commit_reference_filter.rb b/lib/gitlab/markdown/commit_reference_filter.rb index eacdf8a6d37..fcbb2e936a5 100644 --- a/lib/gitlab/markdown/commit_reference_filter.rb +++ b/lib/gitlab/markdown/commit_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces commit references with links. @@ -67,7 +69,7 @@ module Gitlab end def url_for_commit(project, commit) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/cross_project_reference.rb b/lib/gitlab/markdown/cross_project_reference.rb index 66c256c5104..855748fdccc 100644 --- a/lib/gitlab/markdown/cross_project_reference.rb +++ b/lib/gitlab/markdown/cross_project_reference.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # Common methods for ReferenceFilters that support an optional cross-project diff --git a/lib/gitlab/markdown/emoji_filter.rb b/lib/gitlab/markdown/emoji_filter.rb index 6794ab9c897..da10e4d3760 100644 --- a/lib/gitlab/markdown/emoji_filter.rb +++ b/lib/gitlab/markdown/emoji_filter.rb @@ -1,6 +1,7 @@ +require 'action_controller' +require 'gitlab/markdown' require 'gitlab_emoji' require 'html/pipeline/filter' -require 'action_controller' module Gitlab module Markdown diff --git a/lib/gitlab/markdown/external_issue_reference_filter.rb b/lib/gitlab/markdown/external_issue_reference_filter.rb index afd28dd8cf3..f7c43e1ca89 100644 --- a/lib/gitlab/markdown/external_issue_reference_filter.rb +++ b/lib/gitlab/markdown/external_issue_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces external issue tracker references with links. diff --git a/lib/gitlab/markdown/external_link_filter.rb b/lib/gitlab/markdown/external_link_filter.rb index c539e0fb823..29e51b6ade6 100644 --- a/lib/gitlab/markdown/external_link_filter.rb +++ b/lib/gitlab/markdown/external_link_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab diff --git a/lib/gitlab/markdown/issue_reference_filter.rb b/lib/gitlab/markdown/issue_reference_filter.rb index ab6f6bc1cf7..01320f80796 100644 --- a/lib/gitlab/markdown/issue_reference_filter.rb +++ b/lib/gitlab/markdown/issue_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces issue references with links. References to diff --git a/lib/gitlab/markdown/label_reference_filter.rb b/lib/gitlab/markdown/label_reference_filter.rb index 2186f36f854..1e5cb12071e 100644 --- a/lib/gitlab/markdown/label_reference_filter.rb +++ b/lib/gitlab/markdown/label_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces label references with links. @@ -54,7 +56,7 @@ module Gitlab end def url_for_label(project, label) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_issues_path(project.namespace, project, label_name: label.name, only_path: context[:only_path]) diff --git a/lib/gitlab/markdown/merge_request_reference_filter.rb b/lib/gitlab/markdown/merge_request_reference_filter.rb index 884f60f9d53..ecbd263d0e0 100644 --- a/lib/gitlab/markdown/merge_request_reference_filter.rb +++ b/lib/gitlab/markdown/merge_request_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces merge request references with links. References @@ -61,7 +63,7 @@ module Gitlab end def url_for_merge_request(mr, project) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/reference_filter.rb b/lib/gitlab/markdown/reference_filter.rb index 47ee1d99da3..9b293c957d6 100644 --- a/lib/gitlab/markdown/reference_filter.rb +++ b/lib/gitlab/markdown/reference_filter.rb @@ -1,4 +1,5 @@ require 'active_support/core_ext/string/output_safety' +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab @@ -9,7 +10,6 @@ module Gitlab # # Context options: # :project (required) - Current project, ignored if reference is cross-project. - # :reference_class - Custom CSS class added to reference links. # :only_path - Generate path-only links. # # Results: @@ -70,7 +70,7 @@ module Gitlab end def reference_class(type) - "gfm gfm-#{type} #{context[:reference_class]}".strip + "gfm gfm-#{type}" end # Iterate through the document's text nodes, yielding the current node's diff --git a/lib/gitlab/markdown/relative_link_filter.rb b/lib/gitlab/markdown/relative_link_filter.rb index 30f50b82996..8c5cf51bfe1 100644 --- a/lib/gitlab/markdown/relative_link_filter.rb +++ b/lib/gitlab/markdown/relative_link_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'uri' diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index 74b3a8d274f..e368de7d848 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' require 'html/pipeline/sanitization_filter' @@ -66,12 +67,16 @@ module Gitlab def clean_spans lambda do |env| - return unless env[:node_name] == 'span' - return unless env[:node].has_attribute?('class') + node = env[:node] - unless has_ancestor?(env[:node], 'pre') - env[:node].remove_attribute('class') + return unless node.name == 'span' + return unless node.has_attribute?('class') + + unless has_ancestor?(node, 'pre') + node.remove_attribute('class') end + + { node_whitelist: [node] } end end end diff --git a/lib/gitlab/markdown/snippet_reference_filter.rb b/lib/gitlab/markdown/snippet_reference_filter.rb index 92979a356dc..e2cf89cb1d8 100644 --- a/lib/gitlab/markdown/snippet_reference_filter.rb +++ b/lib/gitlab/markdown/snippet_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces snippet references with links. References to @@ -61,7 +63,7 @@ module Gitlab end def url_for_snippet(snippet, project) - h = Rails.application.routes.url_helpers + h = Gitlab::Application.routes.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) end diff --git a/lib/gitlab/markdown/syntax_highlight_filter.rb b/lib/gitlab/markdown/syntax_highlight_filter.rb new file mode 100644 index 00000000000..8597e02f0de --- /dev/null +++ b/lib/gitlab/markdown/syntax_highlight_filter.rb @@ -0,0 +1,45 @@ +require 'gitlab/markdown' +require 'html/pipeline/filter' +require 'rouge/plugins/redcarpet' + +module Gitlab + module Markdown + # HTML Filter to highlight fenced code blocks + # + class SyntaxHighlightFilter < HTML::Pipeline::Filter + include Rouge::Plugins::Redcarpet + + def call + doc.search('pre > code').each do |node| + highlight_node(node) + end + + doc + end + + def highlight_node(node) + language = node.attr('class') + code = node.text + + begin + highlighted = block_code(code, language) + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure + # users can still access an issue/comment/etc. + highlighted = "<pre>#{code}</pre>" + end + + # Replace the parent `pre` element with the entire highlighted block + node.parent.replace(highlighted) + end + + private + + # Override Rouge::Plugins::Redcarpet#rouge_formatter + def rouge_formatter(lexer) + Rouge::Formatters::HTMLGitlab.new( + cssclass: "code highlight js-syntax-highlight #{lexer.tag}") + end + end + end +end diff --git a/lib/gitlab/markdown/table_of_contents_filter.rb b/lib/gitlab/markdown/table_of_contents_filter.rb index 38887c9778c..bbb3bf7fc8b 100644 --- a/lib/gitlab/markdown/table_of_contents_filter.rb +++ b/lib/gitlab/markdown/table_of_contents_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'html/pipeline/filter' module Gitlab diff --git a/lib/gitlab/markdown/task_list_filter.rb b/lib/gitlab/markdown/task_list_filter.rb index c6eb2e2bf6d..2f133ae8500 100644 --- a/lib/gitlab/markdown/task_list_filter.rb +++ b/lib/gitlab/markdown/task_list_filter.rb @@ -1,3 +1,4 @@ +require 'gitlab/markdown' require 'task_list/filter' module Gitlab diff --git a/lib/gitlab/markdown/user_reference_filter.rb b/lib/gitlab/markdown/user_reference_filter.rb index a4aec7a05d1..6f436ea7167 100644 --- a/lib/gitlab/markdown/user_reference_filter.rb +++ b/lib/gitlab/markdown/user_reference_filter.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab module Markdown # HTML filter that replaces user or group references with links. @@ -49,7 +51,7 @@ module Gitlab private def urls - Rails.application.routes.url_helpers + Gitlab::Application.routes.url_helpers end def link_class diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 9b8e783d16c..d94b104bbf8 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -16,16 +16,6 @@ module Gitlab @provider ||= Gitlab::Utils.force_utf8(auth_hash.provider.to_s) end - def info - auth_hash.info - end - - def get_info(key) - value = info.try(key) - Gitlab::Utils.force_utf8(value) if value - value - end - def name @name ||= get_info(:name) || "#{get_info(:first_name)} #{get_info(:last_name)}" end @@ -44,9 +34,19 @@ module Gitlab private + def info + auth_hash.info + end + + def get_info(key) + value = info[key] + Gitlab::Utils.force_utf8(value) if value + value + end + def username_and_email @username_and_email ||= begin - username = get_info(:nickname) || get_info(:username) + username = get_info(:username) || get_info(:nickname) email = get_info(:email) username ||= generate_username(email) if email diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index e836b05ff25..0961bd80421 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,3 +1,5 @@ +require 'gitlab/markdown' + module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor @@ -10,7 +12,7 @@ module Gitlab def analyze(text) references.clear - @text = markdown.render(text.dup) + @text = Gitlab::Markdown.render_without_gfm(text) end %i(user label issue merge_request snippet commit commit_range).each do |type| @@ -21,10 +23,6 @@ module Gitlab private - def markdown - @markdown ||= Redcarpet::Markdown.new(Redcarpet::Render::HTML, GitlabMarkdownHelper::MARKDOWN_OPTIONS) - end - def references @references ||= Hash.new do |references, type| type = type.to_sym @@ -42,7 +40,7 @@ module Gitlab # Returns the results Array for the requested filter type def pipeline_result(filter_type) klass = filter_type.to_s.camelize + 'ReferenceFilter' - filter = "Gitlab::Markdown::#{klass}".constantize + filter = Gitlab::Markdown.const_get(klass) context = { project: project, diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 06245374bc8..2ab2d4af797 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -19,13 +19,15 @@ module Gitlab issues.page(page).per(per_page) when 'merge_requests' merge_requests.page(page).per(per_page) + when 'milestones' + milestones.page(page).per(per_page) else Kaminari.paginate_array([]).page(page).per(per_page) end end def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count end def projects_count @@ -40,6 +42,10 @@ module Gitlab @merge_requests_count ||= merge_requests.count end + def milestones_count + @milestones_count ||= milestones.count + end + def empty? total_count.zero? end @@ -60,6 +66,12 @@ module Gitlab issues.order('updated_at DESC') end + def milestones + milestones = Milestone.where(project_id: limit_project_ids) + milestones = milestones.search(query) + milestones.order('updated_at DESC') + end + def merge_requests merge_requests = MergeRequest.in_projects(limit_project_ids) if query =~ /[#!](\d+)\z/ diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 5209df92795..83f91de810c 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -37,6 +37,11 @@ module Gitlab THEMES.detect { |t| t.id == id } || default end + # Returns the number of defined Themes + def self.count + THEMES.size + end + # Get the default Theme # # Returns a Theme @@ -51,6 +56,19 @@ module Gitlab THEMES.each(&block) end + # Get the Theme for the specified user, or the default + # + # user - User record + # + # Returns a Theme + def self.for_user(user) + if user + by_id(user.theme_id) + else + default + end + end + private def self.default_id diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 11b0d44f340..6f0d02cafd1 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Rails.application.routes.url_helpers + include Gitlab::Application.routes.url_helpers include GitlabRoutingHelper def initialize(type) @@ -23,12 +23,12 @@ module Gitlab def build_issue_url(id) issue = Issue.find(id) - issue_url(issue, host: Gitlab.config.gitlab['url']) + issue_url(issue) end def build_merge_request_url(id) merge_request = MergeRequest.find(id) - merge_request_url(merge_request, host: Gitlab.config.gitlab['url']) + merge_request_url(merge_request) end def build_note_url(id) @@ -37,22 +37,18 @@ module Gitlab namespace_project_commit_url(namespace_id: note.project.namespace, id: note.commit_id, project_id: note.project, - host: Gitlab.config.gitlab['url'], anchor: "note_#{note.id}") elsif note.for_issue? issue = Issue.find(note.noteable_id) issue_url(issue, - host: Gitlab.config.gitlab['url'], anchor: "note_#{note.id}") elsif note.for_merge_request? merge_request = MergeRequest.find(note.noteable_id) merge_request_url(merge_request, - host: Gitlab.config.gitlab['url'], anchor: "note_#{note.id}") elsif note.for_project_snippet? snippet = Snippet.find(note.noteable_id) project_snippet_url(snippet, - host: Gitlab.config.gitlab['url'], anchor: "note_#{note.id}") end end diff --git a/lib/redcarpet/render/gitlab_html.rb b/lib/redcarpet/render/gitlab_html.rb deleted file mode 100644 index f57b56cbdf0..00000000000 --- a/lib/redcarpet/render/gitlab_html.rb +++ /dev/null @@ -1,46 +0,0 @@ -require 'active_support/core_ext/string/output_safety' - -class Redcarpet::Render::GitlabHTML < Redcarpet::Render::HTML - attr_reader :template - alias_method :h, :template - - def initialize(template, color_scheme, options = {}) - @template = template - @color_scheme = color_scheme - @options = options.dup - - @options.reverse_merge!( - # Handled further down the line by Gitlab::Markdown::SanitizationFilter - escape_html: false, - project: @template.instance_variable_get("@project") - ) - - super(options) - end - - def normal_text(text) - ERB::Util.html_escape_once(text) - end - - # Stolen from Rouge::Plugins::Redcarpet as this module is not required - # from Rouge's gem root. - def block_code(code, language) - lexer = Rouge::Lexer.find_fancy(language, code) || Rouge::Lexers::PlainText - - # XXX HACK: Redcarpet strips hard tabs out of code blocks, - # so we assume you're not using leading spaces that aren't tabs, - # and just replace them here. - if lexer.tag == 'make' - code.gsub!(/^ /, "\t") - end - - formatter = Rouge::Formatters::HTMLGitlab.new( - cssclass: "code highlight #{@color_scheme} #{lexer.tag}" - ) - formatter.format(lexer.lex(code)) - end - - def postprocess(full_document) - h.gfm(full_document, @options) - end -end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 41a2f254db6..a80e7e77430 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -37,6 +37,10 @@ web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" mail_room_enabled=false mail_room_pid_path="$pid_path/mail_room.pid" +gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" +gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" +gitlab_git_http_server_repo_root='/home/git/repositories' +gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -72,6 +76,11 @@ check_pids(){ else spid=0 fi + if [ -f "$gitlab_git_http_server_pid_path" ]; then + hpid=$(cat "$gitlab_git_http_server_pid_path") + else + hpid=0 + fi if [ "$mail_room_enabled" = true ]; then if [ -f "$mail_room_pid_path" ]; then mpid=$(cat "$mail_room_pid_path") @@ -85,7 +94,7 @@ check_pids(){ wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_git_http_server_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -120,6 +129,12 @@ check_status(){ else sidekiq_status="-1" fi + if [ $hpid -ne 0 ]; then + kill -0 "$hpid" 2>/dev/null + gitlab_git_http_server_status="$?" + else + gitlab_git_http_server_status="-1" + fi if [ "$mail_room_enabled" = true ]; then if [ $mpid -ne 0 ]; then kill -0 "$mpid" 2>/dev/null @@ -128,7 +143,7 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_git_http_server_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -156,6 +171,13 @@ check_stale_pids(){ exit 1 fi fi + if [ "$hpid" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ]; then + echo "Removing stale gitlab-git-http-server pid. This is most likely caused by gitlab-git-http-server crashing the last time it ran." + if ! rm "$gitlab_git_http_server_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi if [ "$mail_room_enabled" = true ] && [ "$mpid" != "0" ] && [ "$mail_room_status" != "0" ]; then echo "Removing stale MailRoom job dispatcher pid. This is most likely caused by MailRoom crashing the last time it ran." if ! rm "$mail_room_pid_path"; then @@ -168,7 +190,7 @@ check_stale_pids(){ ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -184,6 +206,9 @@ start_gitlab() { if [ "$sidekiq_status" != "0" ]; then echo "Starting GitLab Sidekiq" fi + if [ "$gitlab_git_http_server_status" != "0" ]; then + echo "Starting gitlab-git-http-server" + fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" fi @@ -205,6 +230,17 @@ start_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs start & fi + if [ "$gitlab_git_http_server_status" = "0" ]; then + echo "The gitlab-git-http-server is already running with pid $spid, not restarting" + else + # No need to remove a socket, gitlab-git-http-server does this itself + $app_root/bin/daemon_with_pidfile $gitlab_git_http_server_pid_path \ + $app_root/../gitlab-git-http-server/gitlab-git-http-server \ + $gitlab_git_http_server_options \ + $gitlab_git_http_server_repo_root \ + >> $gitlab_git_http_server_log 2>&1 & + fi + if [ "$mail_room_enabled" = true ]; then # If MailRoom is already running, don't start it again. if [ "$mail_room_status" = "0" ]; then @@ -226,33 +262,27 @@ stop_gitlab() { if [ "$web_status" = "0" ]; then echo "Shutting down GitLab Unicorn" + RAILS_ENV=$RAILS_ENV bin/web stop fi if [ "$sidekiq_status" = "0" ]; then echo "Shutting down GitLab Sidekiq" - fi - if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then - echo "Shutting down GitLab MailRoom" - fi - - # If the Unicorn web server is running, tell it to stop; - if [ "$web_status" = "0" ]; then - RAILS_ENV=$RAILS_ENV bin/web stop - fi - # And do the same thing for the Sidekiq. - if [ "$sidekiq_status" = "0" ]; then RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi - # And do the same thing for the MailRoom. + if [ "$gitlab_git_http_server_status" = "0" ]; then + echo "Shutting down gitlab-git-http-server" + kill -- $(cat $gitlab_git_http_server_pid_path) + fi if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then + echo "Shutting down GitLab MailRoom" RAILS_ENV=$RAILS_ENV bin/mail_room stop fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then printf "\n" break fi @@ -262,6 +292,7 @@ stop_gitlab() { # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. + rm -f "$gitlab_git_http_server_pid_path" if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi @@ -272,7 +303,7 @@ stop_gitlab() { ## Prints the status of GitLab and it's components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_git_http_server_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -286,9 +317,14 @@ print_status() { else printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi + if [ "$gitlab_git_http_server_status" = "0" ]; then + echo "The gitlab-git-http-server with pid $hpid is running." + else + printf "The gitlab-git-http-server is \033[31mnot running\033[0m.\n" + fi if [ "$mail_room_enabled" = true ]; then if [ "$mail_room_status" = "0" ]; then - echo "The GitLab MailRoom email processor with pid $spid is running." + echo "The GitLab MailRoom email processor with pid $mpid is running." else printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi @@ -324,7 +360,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_git_http_server" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index fd70cb7cc74..aab5acaa72c 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,6 +30,16 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +gitlab_git_http_server_pid_path="$pid_path/gitlab-git-http-server.pid" +# The -listenXxx settings determine where gitlab-git-http-server +# listens for connections from NGINX. To listen on localhost:8181, write +# '-listenNetwork tcp -listenAddr localhost:8181'. +# The -authBackend setting tells gitlab-git-http-server where it can reach +# Unicorn. +gitlab_git_http_server_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-git-http-server.socket -authBackend http://127.0.0.1:8080" +gitlab_git_http_server_repo_root="/home/git/repositories" +gitlab_git_http_server_log="$app_root/log/gitlab-git-http-server.log" + # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. # The default is "false" diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index efa0898900f..7218a4d2f20 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -38,10 +38,9 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -## Experimental: gitlab-git-http-server -# upstream gitlab-git-http-server { -# server localhost:8181; -# } +upstream gitlab-git-http-server { + server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +} ## Normal HTTP host server { @@ -114,25 +113,33 @@ server { proxy_pass http://gitlab; } - ## Experimental: send Git HTTP traffic to gitlab-git-http-server instead of Unicorn - # location ~ [-\/\w\.]+\.git\/ { - # ## If you use HTTPS make sure you disable gzip compression - # ## to be safe against BREACH attack. - # # gzip off; - - # ## https://github.com/gitlabhq/gitlabhq/issues/694 - # ## Some requests take more than 30 seconds. - # proxy_read_timeout 300; - # proxy_connect_timeout 300; - # proxy_redirect off; - - # proxy_set_header Host $http_host; - # proxy_set_header X-Real-IP $remote_addr; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-Forwarded-Proto $scheme; - - # proxy_pass http://gitlab-git-http-server; - # } + location ~ [-\/\w\.]+\.git\/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + # gzip off; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + # Do not buffer Git HTTP responses + proxy_buffering off; + + # The following settings only work with NGINX 1.7.11 or newer + # + # # Pass chunked request bodies to gitlab-git-http-server as-is + # proxy_request_buffering off; + # proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_pass http://gitlab-git-http-server; + } ## Enable gzip compression as per rails guide: ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 314525518f1..7dabfba87e2 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -42,10 +42,9 @@ upstream gitlab { server unix:/home/git/gitlab/tmp/sockets/gitlab.socket fail_timeout=0; } -## Experimental: gitlab-git-http-server -# upstream gitlab-git-http-server { -# server localhost:8181; -# } +upstream gitlab-git-http-server { + server unix:/home/git/gitlab/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +} ## Redirects all HTTP traffic to the HTTPS host server { @@ -161,25 +160,33 @@ server { proxy_pass http://gitlab; } - ## Experimental: send Git HTTP traffic to gitlab-git-http-server instead of Unicorn - # location ~ [-\/\w\.]+\.git\/ { - # ## If you use HTTPS make sure you disable gzip compression - # ## to be safe against BREACH attack. - # gzip off; - - # ## https://github.com/gitlabhq/gitlabhq/issues/694 - # ## Some requests take more than 30 seconds. - # proxy_read_timeout 300; - # proxy_connect_timeout 300; - # proxy_redirect off; - - # proxy_set_header Host $http_host; - # proxy_set_header X-Real-IP $remote_addr; - # proxy_set_header X-Forwarded-Ssl on; - # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - # proxy_set_header X-Forwarded-Proto $scheme; - # proxy_pass http://gitlab-git-http-server; - # } + location ~ [-\/\w\.]+\.git\/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + gzip off; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + # Do not buffer Git HTTP responses + proxy_buffering off; + + # The following settings only work with NGINX 1.7.11 or newer + # + # # Pass chunked request bodies to gitlab-git-http-server as-is + # proxy_request_buffering off; + # proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Ssl on; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://gitlab-git-http-server; + } ## Enable gzip compression as per rails guide: ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression diff --git a/lib/support/nginx/gitlab_ci b/lib/support/nginx/gitlab_ci new file mode 100644 index 00000000000..ce179d6f599 --- /dev/null +++ b/lib/support/nginx/gitlab_ci @@ -0,0 +1,41 @@ +# GITLAB CI +server { + listen 80 default_server; # e.g., listen 192.168.1.1:80; + server_name YOUR_CI_SERVER_FQDN; # e.g., server_name source.example.com; + + access_log /var/log/nginx/gitlab_ci_access.log; + error_log /var/log/nginx/gitlab_ci_error.log; + + # expose API to fix runners + location /api { + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + + # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN + resolver 8.8.8.8 8.8.4.4; + proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; + } + + # expose build endpoint to allow trigger builds + location ~ ^/projects/\d+/build$ { + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + proxy_set_header X-Real-IP $remote_addr; + + # You need to specify your DNS servers that are able to resolve YOUR_GITLAB_SERVER_FQDN + resolver 8.8.8.8 8.8.4.4; + proxy_pass $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; + } + + # redirect all other CI requests + location / { + return 301 $scheme://YOUR_GITLAB_SERVER_FQDN/ci$request_uri; + } + + # adjust this to match the largest build log your runners might submit, + # set to 0 to disable limit + client_max_body_size 10m; +}
\ No newline at end of file diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/lib/tasks/ci/.gitkeep diff --git a/lib/tasks/ci/cleanup.rake b/lib/tasks/ci/cleanup.rake new file mode 100644 index 00000000000..2f4d11bd942 --- /dev/null +++ b/lib/tasks/ci/cleanup.rake @@ -0,0 +1,8 @@ +namespace :ci do + namespace :cleanup do + desc "GitLab CI | Clean running builds" + task builds: :environment do + Ci::Build.running.update_all(status: 'canceled') + end + end +end diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake new file mode 100644 index 00000000000..e7d41874a11 --- /dev/null +++ b/lib/tasks/ci/migrate.rake @@ -0,0 +1,63 @@ +namespace :ci do + desc 'GitLab | Import and migrate CI database' + task migrate: :environment do + unless ENV['force'] == 'yes' + puts "This will truncate all CI tables and restore it from provided backup." + puts "You will lose any previous CI data stored in the database." + ask_to_continue + puts "" + end + + Rake::Task["ci:migrate:db"].invoke + Rake::Task["ci:migrate:autoincrements"].invoke + Rake::Task["ci:migrate:tags"].invoke + Rake::Task["ci:migrate:services"].invoke + end + + namespace :migrate do + desc 'GitLab | Import CI database' + task db: :environment do + if ENV["CI_DUMP"].nil? + puts "No CI SQL dump specified:" + puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" + exit 1 + end + + ci_dump = ENV["CI_DUMP"] + unless File.exists?(ci_dump) + puts "The specified sql dump doesn't exist!" + exit 1 + end + + ::Ci::Migrate::Database.new.restore(ci_dump) + end + + desc 'GitLab | Migrate CI tags' + task tags: :environment do + ::Ci::Migrate::Tags.new.restore + end + + desc 'GitLab | Migrate CI auto-increments' + task autoincrements: :environment do + c = ActiveRecord::Base.connection + c.tables.select { |t| t.start_with?('ci_') }.each do |table| + result = c.select_one("SELECT id FROM #{table} ORDER BY id DESC LIMIT 1") + if result + ai_val = result['id'].to_i + 1 + puts "Resetting auto increment ID for #{table} to #{ai_val}" + if c.adapter_name == 'PostgreSQL' + c.execute("ALTER SEQUENCE #{table}_id_seq RESTART WITH #{ai_val}") + else + c.execute("ALTER TABLE #{table} AUTO_INCREMENT = #{ai_val}") + end + end + end + end + + desc 'GitLab | Migrate CI services' + task services: :environment do + c = ActiveRecord::Base.connection + c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") + end + end +end diff --git a/lib/tasks/ci/schedule_builds.rake b/lib/tasks/ci/schedule_builds.rake new file mode 100644 index 00000000000..49435504c67 --- /dev/null +++ b/lib/tasks/ci/schedule_builds.rake @@ -0,0 +1,6 @@ +namespace :ci do + desc "GitLab CI | Clean running builds" + task schedule_builds: :environment do + Ci::Scheduler.new.perform + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 4c73f90bbf2..f20c7f71ba5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -11,6 +11,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke + Rake::Task["gitlab:backup:builds:create"].invoke backup = Backup::Manager.new backup.pack @@ -30,6 +31,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") + Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -73,6 +75,25 @@ namespace :gitlab do end end + namespace :builds do + task create: :environment do + $progress.puts "Dumping builds ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("builds") + $progress.puts "[SKIPPED]".cyan + else + Backup::Builds.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring builds ... ".blue + Backup::Builds.new.restore + $progress.puts "done".green + end + end + namespace :uploads do task create: :environment do $progress.puts "Dumping uploads ... ".blue diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2b9688c1b40..b8eb13a4fea 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -278,7 +278,7 @@ namespace :gitlab do fix_and_rerun end end - + def check_uploads print "Uploads directory setup correctly? ... " @@ -331,15 +331,18 @@ namespace :gitlab do end def check_redis_version - print "Redis version >= 2.0.0? ... " + min_redis_version = "2.4.0" + print "Redis version >= #{min_redis_version}? ... " redis_version = run(%W(redis-cli --version)) - if redis_version.try(:match, /redis-cli 2.\d.\d/) || redis_version.try(:match, /redis-cli 3.\d.\d/) + redis_version = redis_version.try(:match, /redis-cli (.*)/) + if redis_version && + (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) puts "yes".green else puts "no".red try_fixing_it( - "Update your redis server to a version >= 2.0.0" + "Update your redis server to a version >= #{min_redis_version}" ) for_more_information( "gitlab-public-wiki/wiki/Trouble-Shooting-Guide in section sidekiq" @@ -488,7 +491,7 @@ namespace :gitlab do else puts "wrong or missing hooks".red try_fixing_it( - sudo_gitlab("#{gitlab_shell_path}/bin/create-hooks"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake new file mode 100644 index 00000000000..39541c0b9c6 --- /dev/null +++ b/lib/tasks/services.rake @@ -0,0 +1,98 @@ +services_template = <<-ERB +# Services + +<% services.each do |service| %> +## <%= service[:title] %> + + +<% unless service[:description].blank? %> +<%= service[:description] %> +<% end %> + + +### Create/Edit <%= service[:title] %> service + +Set <%= service[:title] %> service for a project. +<% unless service[:help].blank? %> + +> <%= service[:help].gsub("\n", ' ') %> + +<% end %> + +``` +PUT /projects/:id/services/<%= service[:dashed_name] %> + +``` + +Parameters: + +<% service[:params].each do |param| %> +- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)" %><%= [" -", param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %> + +<% end %> + +### Delete <%= service[:title] %> service + +Delete <%= service[:title] %> service for a project. + +``` +DELETE /projects/:id/services/<%= service[:dashed_name] %> + +``` + +### Get <%= service[:title] %> service settings + +Get <%= service[:title] %> service settings for a project. + +``` +GET /projects/:id/services/<%= service[:dashed_name] %> + +``` + +<% end %> +ERB + +namespace :services do + task doc: :environment do + services = Service.available_services_names.map do |s| + service_start = Time.now + klass = "#{s}_service".classify.constantize + + service = klass.new + + service_hash = {} + + service_hash[:title] = service.title + service_hash[:dashed_name] = s.dasherize + service_hash[:description] = service.description + service_hash[:help] = service.help + service_hash[:params] = service.fields.map do |p| + param_hash = {} + + param_hash[:name] = p[:name] + param_hash[:description] = p[:placeholder] || p[:title] + param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v| + v.class == ActiveRecord::Validations::PresenceValidator + end + + param_hash + end.sort_by { |p| p[:required] ? 0 : 1 } + + puts "Collected data for: #{service.title}, #{Time.now-service_start}" + service_hash + end + + doc_start = Time.now + doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') + + result = ERB.new(services_template, 0 , '>') + .result(OpenStruct.new(services: services).instance_eval { binding }) + + File.open(doc_path, 'w') do |f| + f.write result + end + + puts "write a new service.md to: #{doc_path.to_s}, #{Time.now-doc_start}" + + end +end |