diff options
author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-06-20 11:38:46 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2016-06-20 11:38:46 +0300 |
commit | 9510d31b4dd5955af2941a20a09d2dff6a55249a (patch) | |
tree | 6eff16ed12a897f19af2f0ab9d481d1c39804d09 /lib | |
parent | 44b00a1ebbfeaa095343f55f6c12dcbc65b85924 (diff) | |
parent | 44b8b77e02423ce97f9abe80e0335f4f4c453c83 (diff) |
Merge branch 'master' into refactor/ci-config-add-entry-error
* master: (345 commits)
use rails root join
fixed a couple of errors spotted in production
Fix RangeError exceptions when referring to issues or merge requests outside of max database values
Fix bug in `WikiLinkFilter`.
Small frontend code fixes and restore 8a2d88f commit
Warn about admin privilege to disable GitHub Webhooks
Listing GH Webhooks doesn't stop import process for non GH admin users
fixup! updated docs for api endpoint award emoji
Update CHANGELOG
Ensure Todos counters doesn't count Todos for projects pending delete
Add endpoints for award emoji on notes
Sort API endpoints and implement feedback
Add endpoints for Award Emoji
Fixed issue with assignee dropdown not selecting correctly
Removed update method Re-structured controller spec Renamed issuable param to issuable_id
Fix clibpoard buttons on "Check out branch" modal.
Track method call times/counts as a single metric
Cache todo counters (pending/done)
Fix a 'wrong number of arguments' error
Added missing mount point for Sidekiq Metrics API, after it got lost on rebase.
...
Diffstat (limited to 'lib')
43 files changed, 1503 insertions, 122 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cd909f6115..0e7a1cc2623 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -26,38 +26,40 @@ module API # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers - mount ::API::Groups + mount ::API::AwardEmoji + mount ::API::Branches + mount ::API::Builds + mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::DeployKeys + mount ::API::Files + mount ::API::Gitignores mount ::API::GroupMembers - mount ::API::Users - mount ::API::Projects - mount ::API::Repositories + mount ::API::Groups + mount ::API::Internal mount ::API::Issues - mount ::API::Milestones - mount ::API::Session + mount ::API::Keys + mount ::API::Labels + mount ::API::Licenses mount ::API::MergeRequests + mount ::API::Milestones + mount ::API::Namespaces mount ::API::Notes - mount ::API::Internal - mount ::API::SystemHooks - mount ::API::ProjectSnippets - mount ::API::ProjectMembers - mount ::API::DeployKeys mount ::API::ProjectHooks + mount ::API::ProjectMembers + mount ::API::ProjectSnippets + mount ::API::Projects + mount ::API::Repositories + mount ::API::Runners mount ::API::Services - mount ::API::Files - mount ::API::Commits - mount ::API::CommitStatuses - mount ::API::Namespaces - mount ::API::Branches - mount ::API::Labels + mount ::API::Session mount ::API::Settings - mount ::API::Keys + mount ::API::SidekiqMetrics + mount ::API::Subscriptions + mount ::API::SystemHooks mount ::API::Tags mount ::API::Triggers - mount ::API::Builds + mount ::API::Users mount ::API::Variables - mount ::API::Runners - mount ::API::Licenses - mount ::API::Subscriptions - mount ::API::Gitignores end end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb new file mode 100644 index 00000000000..985590312e3 --- /dev/null +++ b/lib/api/award_emoji.rb @@ -0,0 +1,116 @@ +module API + class AwardEmoji < Grape::API + before { authenticate! } + AWARDABLES = [Issue, MergeRequest] + + resource :projects do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.to_s.underscore.pluralize + awardable_id_string = "#{awardable_type.to_s.underscore}_id" + + [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + + # Get a list of project +awardable+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji + get endpoint do + if can_read_awardable? + awards = paginate(awardable.award_emoji) + present awards, with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Get a specific award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_id (required) - The ID of the award + # Example Request: + # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + # Award a new Emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or mr + # name (required) - The name of a award_emoji (without colons) + # Example Request: + # POST /projects/:id/issues/:awardable_id/award_emoji + post endpoint do + required_attributes! [:name] + + not_found!('Award Emoji') unless can_read_awardable? + + award = awardable.award_emoji.new(name: params[:name], user: current_user) + + if award.save + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + # Delete a +awardables+ award emoji + # + # Parameters: + # id (required) - The ID of a project + # awardable_id (required) - The ID of an issue or MR + # award_emoji_id (required) - The ID of an award emoji + # Example Request: + # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + end + + helpers do + def can_read_awardable? + ability = "read_#{awardable.class.to_s.underscore}".to_sym + + can?(current_user, ability, awardable) + end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + noteable.notes.find(params[:note_id]) + else + noteable + end + end + end + + def noteable + if params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + else + user_project.merge_requests.find(params[:merge_request_id]) + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index cc29c7ef428..2e397643ed1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -225,6 +225,14 @@ module API expose(:downvote?) { |note| false } end + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + class MRNote < Grape::Entity expose :note expose :author, using: Entities::UserBasic diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index de5959e3aae..77e407b54c5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -9,9 +9,13 @@ module API [ true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON' ].include?(value) end + def find_user_by_private_token + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + end + def current_user - private_token = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - @current_user ||= (User.find_by(authentication_token: private_token) || doorkeeper_guard) + @current_user ||= (find_user_by_private_token || doorkeeper_guard) unless @current_user && Gitlab::UserAccess.allowed?(@current_user) return nil @@ -33,7 +37,7 @@ module API identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] # Regex for integers - if !!(identifier =~ /^[0-9]+$/) + if !!(identifier =~ /\A[0-9]+\z/) identifier.to_i else identifier diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d4fcfd3d4d3..8bfa998dc53 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -144,7 +144,7 @@ module API helpers do def noteable_read_ability_name(noteable) - "read_#{noteable.class.to_s.underscore.downcase}".to_sym + "read_#{noteable.class.to_s.underscore}".to_sym end end end diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb new file mode 100644 index 00000000000..d3d6827dc54 --- /dev/null +++ b/lib/api/sidekiq_metrics.rb @@ -0,0 +1,90 @@ +require 'sidekiq/api' + +module API + class SidekiqMetrics < Grape::API + before { authenticated_as_admin! } + + helpers do + def queue_metrics + Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + hash[queue.name] = { + backlog: queue.size, + latency: queue.latency.to_i + } + end + end + + def process_metrics + Sidekiq::ProcessSet.new.map do |process| + { + hostname: process['hostname'], + pid: process['pid'], + tag: process['tag'], + started_at: Time.at(process['started_at']), + queues: process['queues'], + labels: process['labels'], + concurrency: process['concurrency'], + busy: process['busy'] + } + end + end + + def job_stats + stats = Sidekiq::Stats.new + { + processed: stats.processed, + failed: stats.failed, + enqueued: stats.enqueued + } + end + end + + # Get Sidekiq Queue metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/queue_metrics + # + get 'sidekiq/queue_metrics' do + { queues: queue_metrics } + end + + # Get Sidekiq Process metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/process_metrics + # + get 'sidekiq/process_metrics' do + { processes: process_metrics } + end + + # Get Sidekiq Job statistics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/job_stats + # + get 'sidekiq/job_stats' do + { jobs: job_stats } + end + + # Get Sidekiq Compound metrics. Includes all previous metrics + # + # Parameters: + # None + # + # Example: + # GET /sidekiq/compound_metrics + # + get 'sidekiq/compound_metrics' do + { queues: queue_metrics, processes: process_metrics, jobs: job_stats } + end + end +end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 4815bafe238..81d66271136 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -218,8 +218,9 @@ module Banzai nodes.each do |node| node.to_html.scan(regex) do project = $~[:project] || current_project_path + symbol = $~[object_sym] - refs[project] << $~[object_sym] + refs[project] << symbol if object_class.reference_valid?(symbol) end end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 37a2779d453..1bb6d6bba87 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -29,7 +29,7 @@ module Banzai return if html_attr.blank? html_attr.value = apply_rewrite_rules(html_attr.value) - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index a051748cf43..c52d4d63382 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -54,7 +54,7 @@ module Ci job = @jobs[name.to_sym] return [] unless job - job.fetch(:variables, []) + job[:variables] || [] end private @@ -204,12 +204,12 @@ module Ci raise ValidationError, "#{name} job: tags parameter should be an array of strings" end - if job[:only] && !validate_array_of_strings(job[:only]) - raise ValidationError, "#{name} job: only parameter should be an array of strings" + if job[:only] && !validate_array_of_strings_or_regexps(job[:only]) + raise ValidationError, "#{name} job: only parameter should be an array of strings or regexps" end - if job[:except] && !validate_array_of_strings(job[:except]) - raise ValidationError, "#{name} job: except parameter should be an array of strings" + if job[:except] && !validate_array_of_strings_or_regexps(job[:except]) + raise ValidationError, "#{name} job: except parameter should be an array of strings or regexps" end if job[:allow_failure] && !validate_boolean(job[:allow_failure]) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e0b3f14d384..42232b7129d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,11 +15,11 @@ module ContainerRegistry end def repository_tags(name) - @faraday.get("/v2/#{name}/tags/list").body + response_body @faraday.get("/v2/#{name}/tags/list") end def repository_manifest(name, reference) - @faraday.get("/v2/#{name}/manifests/#{reference}").body + response_body @faraday.get("/v2/#{name}/manifests/#{reference}") end def repository_tag_digest(name, reference) @@ -34,7 +34,7 @@ module ContainerRegistry def blob(name, digest, type = nil) headers = {} headers['Accept'] = type if type - @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + response_body @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers) end def delete_blob(name, digest) @@ -47,6 +47,7 @@ module ContainerRegistry conn.request :json conn.headers['Accept'] = MANIFEST_VERSION + conn.response :json, content_type: 'application/json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v2+json' @@ -59,5 +60,9 @@ module ContainerRegistry conn.adapter :net_http end + + def response_body(response) + response.body if response.success? + end end end diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb index 095d0ac1047..4d9a508796a 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/node/legacy_validation_helpers.rb @@ -15,6 +15,10 @@ module Gitlab values.is_a?(Array) && values.all? { |value| validate_string(value) } end + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all? { |value| validate_string_or_regexp(value) } + end + def validate_variables(variables) variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } @@ -24,6 +28,19 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_string_or_regexp(value) + return true if value.is_a?(Symbol) + return false unless value.is_a?(String) + + if value.first == '/' && value.last == '/' + Regexp.new(value[1...-1]) + else + true + end + rescue RegexpError + false + end + def validate_environment(value) value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 5e7532f57ae..28c34429c1f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -36,7 +36,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], - import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], + import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d76ecb54017..078609c86f1 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -1,5 +1,10 @@ module Gitlab module Database + # The max value of INTEGER type is the same between MySQL and PostgreSQL: + # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html + # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html + MAX_INT_VALUE = 2147483647 + def self.adapter_name connection.adapter_name end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index dd3ff0ab18b..dec20d8659b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -28,65 +28,79 @@ module Gitlab # Updates the value of a column in batches. # # This method updates the table in batches of 5% of the total row count. - # Any data inserted while running this method (or after it has finished - # running) is _not_ updated automatically. + # This method will continue updating rows until no rows remain. + # + # When given a block this method will yield two values to the block: + # + # 1. An instance of `Arel::Table` for the table that is being updated. + # 2. The query to run as an Arel object. + # + # By supplying a block one can add extra conditions to the queries being + # executed. Note that the same block is used for _all_ queries. + # + # Example: + # + # update_column_in_batches(:projects, :foo, 10) do |table, query| + # query.where(table[:some_column].eq('hello')) + # end + # + # This would result in this method updating only rows where + # `projects.some_column` equals "hello". # # table - The name of the table. # column - The name of the column to update. # value - The value for the column. + # + # Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop + # determines this method to be too complex while there's no way to make it + # less "complex" without introducing extra methods (which actually will + # make things _more_ complex). + # + # rubocop: disable Metrics/AbcSize def update_column_in_batches(table, column, value) - quoted_table = quote_table_name(table) - quoted_column = quote_column_name(column) - - ## - # Workaround for #17711 - # - # It looks like for MySQL `ActiveRecord::Base.conntection.quote(true)` - # returns correct value (1), but `ActiveRecord::Migration.new.quote` - # returns incorrect value ('true'), which causes migrations to fail. - # - quoted_value = connection.quote(value) - processed = 0 - - total = exec_query("SELECT COUNT(*) AS count FROM #{quoted_table}"). - to_hash. - first['count']. - to_i + table = Arel::Table.new(table) + + count_arel = table.project(Arel.star.count.as('count')) + count_arel = yield table, count_arel if block_given? + + total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + + return if total == 0 # Update in batches of 5% until we run out of any rows to update. batch_size = ((total / 100.0) * 5.0).ceil + start_arel = table.project(table[:id]).order(table[:id].asc).take(1) + start_arel = yield table, start_arel if block_given? + start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i + loop do - start_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed} - }).to_hash.first - - # There are no more rows to process - break unless start_row - - stop_row = exec_query(%Q{ - SELECT id - FROM #{quoted_table} - ORDER BY id ASC - LIMIT 1 OFFSET #{processed + batch_size} - }).to_hash.first - - query = %Q{ - UPDATE #{quoted_table} - SET #{quoted_column} = #{quoted_value} - WHERE id >= #{start_row['id']} - } + stop_arel = table.project(table[:id]). + where(table[:id].gteq(start_id)). + order(table[:id].asc). + take(1). + skip(batch_size) + + stop_arel = yield table, stop_arel if block_given? + stop_row = exec_query(stop_arel.to_sql).to_hash.first + + update_arel = Arel::UpdateManager.new(ActiveRecord::Base). + table(table). + set([[table[column], value]]). + where(table[:id].gteq(start_id)) if stop_row - query += " AND id < #{stop_row['id']}" + stop_id = stop_row['id'].to_i + start_id = stop_id + update_arel = update_arel.where(table[:id].lt(stop_id)) end - execute(query) + update_arel = yield table, update_arel if block_given? + + execute(update_arel.to_sql) - processed += batch_size + # There are no more rows left to update. + break unless stop_row end end @@ -95,9 +109,9 @@ module Gitlab # This method runs the following steps: # # 1. Add the column with a default value of NULL. - # 2. Update all existing rows in batches. - # 3. Change the default value of the column to the specified value. - # 4. Update any remaining rows. + # 2. Change the default value of the column to the specified value. + # 3. Update all existing rows in batches. + # 4. Set a `NOT NULL` constraint on the column if desired (the default). # # These steps ensure a column can be added to a large and commonly used # table without locking the entire table for the duration of the table @@ -109,7 +123,10 @@ module Gitlab # default - The default value for the column. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. - def add_column_with_default(table, column, type, default:, allow_null: false) + # + # This method can also take a block which is passed directly to the + # `update_column_in_batches` method. + def add_column_with_default(table, column, type, default:, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -125,11 +142,9 @@ module Gitlab end begin - transaction do - update_column_in_batches(table, column, default) + update_column_in_batches(table, column, default, &block) - change_column_null(table, column, false) unless allow_null - end + change_column_null(table, column, false) unless allow_null # We want to rescue _all_ exceptions here, even those that don't inherit # from StandardError. rescue Exception => error # rubocop: disable all diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index e5cf66a0371..2286ac8829c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -66,8 +66,7 @@ module Gitlab end def import_pull_requests - hooks = client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) - disable_webhooks(hooks) + disable_webhooks pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) @@ -90,14 +89,14 @@ module Gitlab raise Projects::ImportService::Error, e.message ensure clean_up_restored_branches(branches_removed) - clean_up_disabled_webhooks(hooks) + clean_up_disabled_webhooks end - def disable_webhooks(hooks) + def disable_webhooks update_webhooks(hooks, active: false) end - def clean_up_disabled_webhooks(hooks) + def clean_up_disabled_webhooks update_webhooks(hooks, active: true) end @@ -107,6 +106,20 @@ module Gitlab end end + def hooks + @hooks ||= + begin + client.hooks(repo).map { |raw| HookFormatter.new(raw) }.select(&:valid?) + + # The GitHub Repository Webhooks API returns 404 for users + # without admin access to the repository when listing hooks. + # In this case we just want to return gracefully instead of + # spitting out an error and stop the import process. + rescue Octokit::NotFound + [] + end + end + def restore_branches(branches) branches.each do |name, sha| client.create_ref(repo, "refs/heads/#{name}", sha) diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb index 77c33db4b59..3d0418261bb 100644 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ b/lib/gitlab/gitlab_import/project_creator.rb @@ -11,7 +11,7 @@ module Gitlab end def execute - project = ::Projects::CreateService.new( + ::Projects::CreateService.new( current_user, name: repo["name"], path: repo["path"], @@ -22,8 +22,6 @@ module Gitlab import_source: repo["path_with_namespace"], import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") ).execute - - project end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb new file mode 100644 index 00000000000..99cf85d9a3b --- /dev/null +++ b/lib/gitlab/import_export.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + extend self + + VERSION = '0.1.0' + + def export_path(relative_path:) + File.join(storage_path, relative_path) + end + + def storage_path + File.join(Settings.shared['path'], 'tmp/project_exports') + end + + def project_filename + "project.json" + end + + def project_bundle_filename + "project.bundle" + end + + def config_file + Rails.root.join('lib/gitlab/import_export/import_export.yml') + end + + def version_filename + 'VERSION' + end + + def version + VERSION + end + + def reset_tokens? + true + end + end +end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb new file mode 100644 index 00000000000..d230de781d5 --- /dev/null +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -0,0 +1,47 @@ +module Gitlab + module ImportExport + class AttributesFinder + + def initialize(included_attributes:, excluded_attributes:, methods:) + @included_attributes = included_attributes || {} + @excluded_attributes = excluded_attributes || {} + @methods = methods || {} + end + + def find(model_object) + parsed_hash = find_attributes_only(model_object) + parsed_hash.empty? ? model_object : { model_object => parsed_hash } + end + + def parse(model_object) + parsed_hash = find_attributes_only(model_object) + yield parsed_hash unless parsed_hash.empty? + end + + def find_included(value) + key = key_from_hash(value) + @included_attributes[key].nil? ? {} : { only: @included_attributes[key] } + end + + def find_excluded(value) + key = key_from_hash(value) + @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] } + end + + def find_method(value) + key = key_from_hash(value) + @methods[key].nil? ? {} : { methods: @methods[key] } + end + + private + + def find_attributes_only(value) + find_included(value).merge(find_excluded(value)).merge(find_method(value)) + end + + def key_from_hash(value) + value.is_a?(Hash) ? value.keys.first : value + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb new file mode 100644 index 00000000000..78664f076eb --- /dev/null +++ b/lib/gitlab/import_export/command_line_util.rb @@ -0,0 +1,40 @@ +module Gitlab + module ImportExport + module CommandLineUtil + def tar_czf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'czf') + end + + def untar_zxf(archive:, dir:) + untar_with_options(archive: archive, dir: dir, options: 'zxf') + end + + def git_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) + end + + def git_unbundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) + end + + private + + def tar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir} .)) + end + + def untar_with_options(archive:, dir:, options:) + execute(%W(tar -#{options} #{archive} -C #{dir})) + end + + def execute(cmd) + _output, status = Gitlab::Popen.popen(cmd) + status.zero? + end + + def git_bin_path + Gitlab.config.git.bin_path + end + end + end +end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb new file mode 100644 index 00000000000..e341c4d9cf8 --- /dev/null +++ b/lib/gitlab/import_export/error.rb @@ -0,0 +1,5 @@ +module Gitlab + module ImportExport + class Error < StandardError; end + end +end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb new file mode 100644 index 00000000000..0e70d9282d5 --- /dev/null +++ b/lib/gitlab/import_export/file_importer.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class FileImporter + include Gitlab::ImportExport::CommandLineUtil + + def self.import(*args) + new(*args).import + end + + def initialize(archive_file:, shared:) + @archive_file = archive_file + @shared = shared + end + + def import + FileUtils.mkdir_p(@shared.export_path) + decompress_archive + rescue => e + @shared.error(e) + false + end + + private + + def decompress_archive + untar_zxf(archive: @archive_file, dir: @shared.export_path) + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml new file mode 100644 index 00000000000..164ab6238c4 --- /dev/null +++ b/lib/gitlab/import_export/import_export.yml @@ -0,0 +1,54 @@ +# Model relationships to be included in the project import/export +project_tree: + - issues: + - notes: + :author + - :labels + - :milestones + - snippets: + - notes: + :author + - :releases + - :events + - project_members: + - :user + - merge_requests: + - notes: + :author + - :merge_request_diff + - pipelines: + - notes: + :author + - :statuses + - :variables + - :triggers + - :deploy_keys + - :services + - :hooks + - :protected_branches + +# Only include the following attributes for the models specified. +included_attributes: + project: + - :description + - :issues_enabled + - :merge_requests_enabled + - :wiki_enabled + - :snippets_enabled + - :visibility_level + - :archived + user: + - :id + - :email + - :username + author: + - :name + +# Do not include the following attributes for the models specified. +excluded_attributes: + snippets: + - :expired_at + +methods: + statuses: + - :type
\ No newline at end of file diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb new file mode 100644 index 00000000000..d209e04f7be --- /dev/null +++ b/lib/gitlab/import_export/importer.rb @@ -0,0 +1,64 @@ +module Gitlab + module ImportExport + class Importer + + def initialize(project) + @archive_file = project.import_source + @current_user = project.creator + @project = project + @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + end + + def execute + Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + shared: @shared) + if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore) + project_tree.restored_project + else + raise Projects::ImportService::Error.new(@shared.errors.join(', ')) + end + end + + private + + def check_version! + Gitlab::ImportExport::VersionChecker.check!(shared: @shared) + end + + def project_tree + @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user, + shared: @shared, + project: @project) + end + + def repo_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, + shared: @shared, + project: project_tree.restored_project) + end + + def wiki_restorer + Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + shared: @shared, + project: ProjectWiki.new(project_tree.restored_project), + wiki: true) + end + + def uploads_restorer + Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def path_with_namespace + File.join(@project.namespace.path, @project.path) + end + + def repo_path + File.join(@shared.export_path, 'project.bundle') + end + + def wiki_repo_path + File.join(@shared.export_path, 'project.wiki.bundle') + end + end + end +end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb new file mode 100644 index 00000000000..c569a35a48b --- /dev/null +++ b/lib/gitlab/import_export/members_mapper.rb @@ -0,0 +1,68 @@ +module Gitlab + module ImportExport + class MembersMapper + + attr_reader :missing_author_ids + + def initialize(exported_members:, user:, project:) + @exported_members = exported_members + @user = user + @project = project + @missing_author_ids = [] + + # This needs to run first, as second call would be from #map + # which means project members already exist. + ensure_default_member! + end + + def map + @map ||= + begin + @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + existing_user = User.where(find_project_user_query(member)).first + old_user_id = member['user']['id'] + if existing_user && add_user_as_team_member(existing_user, member) + hash[old_user_id] = existing_user.id + end + hash + end + end + end + + def default_user_id + @user.id + end + + private + + def missing_keys_tracking_hash + Hash.new do |_, key| + @missing_author_ids << key + default_user_id + end + end + + def ensure_default_member! + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) + end + + def add_user_as_team_member(existing_user, member) + member['user'] = existing_user + + ProjectMember.create(member_hash(member)).persisted? + end + + def member_hash(member) + member.except('id').merge(source_id: @project.id, importing: true) + end + + def find_project_user_query(member) + user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + end + + def user_arel + @user_arel ||= User.arel_table + end + end + end +end diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb new file mode 100644 index 00000000000..89388d1984b --- /dev/null +++ b/lib/gitlab/import_export/project_creator.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class ProjectCreator + + def initialize(namespace_id, current_user, file, project_path) + @namespace_id = namespace_id + @current_user = current_user + @file = file + @project_path = project_path + end + + def execute + ::Projects::CreateService.new( + @current_user, + name: @project_path, + path: @project_path, + namespace_id: @namespace_id, + import_type: "gitlab_project", + import_source: @file + ).execute + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb new file mode 100644 index 00000000000..dd71b92c522 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -0,0 +1,105 @@ +module Gitlab + module ImportExport + class ProjectTreeRestorer + + def initialize(user:, shared:, project:) + @path = File.join(shared.export_path, 'project.json') + @user = user + @shared = shared + @project = project + end + + def restore + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + @project_members = @tree_hash.delete('project_members') + create_relations + rescue => e + @shared.error(e) + false + end + + def restored_project + @restored_project ||= restore_project + end + + private + + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project: restored_project) + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project. + def create_relations + saved = [] + default_relation_list.each do |relation| + next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present? + + create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash) + + relation_key = relation.is_a?(Hash) ? relation.keys.first : relation + relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s]) + saved << restored_project.update_attribute(relation_key, relation_hash) + end + saved.all? + end + + def default_relation_list + Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + model.is_a?(Hash) && model[:project_members] + end + end + + def restore_project + return @project unless @tree_hash + + project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } + @project.update(project_params) + @project + end + + # Given a relation hash containing one or more models and its relationships, + # loops through each model and each object from a model type and + # and assigns its correspondent attributes hash from +tree_hash+ + # Example: + # +relation_key+ issues, loops through the list of *issues* and for each individual + # issue, finds any subrelations such as notes, creates them and assign them back to the hash + def create_sub_relations(relation, tree_hash) + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| + relation.values.flatten.each do |sub_relation| + relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation) + relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank? + end + end + end + + def assign_relation_hash(relation_item, sub_relation) + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + [relation_hash, sub_relation] + end + + def create_relation(relation, relation_hash_list) + relation_array = [relation_hash_list].flatten.map do |relation_hash| + Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, + relation_hash: relation_hash.merge('project_id' => restored_project.id), + members_mapper: members_mapper, + user: @user) + end + + relation_hash_list.is_a?(Array) ? relation_array : relation_array.first + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb new file mode 100644 index 00000000000..9153088e966 --- /dev/null +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -0,0 +1,29 @@ +module Gitlab + module ImportExport + class ProjectTreeSaver + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(full_path, project_json_tree) + true + rescue => e + @shared.error(e) + false + end + + private + + def project_json_tree + @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + end + end + end +end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb new file mode 100644 index 00000000000..19defd8f03a --- /dev/null +++ b/lib/gitlab/import_export/reader.rb @@ -0,0 +1,117 @@ +module Gitlab + module ImportExport + class Reader + + attr_reader :tree + + def initialize(shared:) + @shared = shared + config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys + @tree = config_hash[:project_tree] + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes], + excluded_attributes: config_hash[:excluded_attributes], + methods: config_hash[:methods]) + end + + # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # for outputting a project in JSON format, including its relations and sub relations. + def project_tree + @attributes_finder.find_included(:project).merge(include: build_hash(@tree)) + rescue => e + @shared.error(e) + false + end + + private + + # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html + # + # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file + def build_hash(model_list) + model_list.map do |model_objects| + if model_objects.is_a?(Hash) + build_json_config_hash(model_objects) + else + @attributes_finder.find(model_objects) + end + end + end + + # Called when the model is actually a hash containing other relations (more models) + # Returns the config in the right format for calling +to_json+ + # +model_object_hash+ - A model relationship such as: + # {:merge_requests=>[:merge_request_diff, :notes]} + def build_json_config_hash(model_object_hash) + @json_config_hash = {} + + model_object_hash.values.flatten.each do |model_object| + current_key = model_object_hash.keys.first + + @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } + + handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) + end + @json_config_hash + end + + + # If the model is a hash, process the sub_models, which could also be hashes + # If there is a list, add to an existing array, otherwise use hash syntax + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + + # Creates or adds to an existing hash an individual model or list + # +current_key+ main model that will be a key in the hash + # +model_object+ model or list of models to include in the hash + def handle_model_object(current_key, model_object) + if @json_config_hash[current_key] + add_model_value(current_key, model_object) + else + create_model_value(current_key, model_object) + end + end + + # Constructs a new hash that will hold the configuration for that particular object + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def create_model_value(current_key, value) + parsed_hash = { include: value } + + @attributes_finder.parse(value) do |hash| + parsed_hash = { include: hash_or_merge(value, hash) } + end + @json_config_hash[current_key] = parsed_hash + end + + # Adds new model configuration to an existing hash with key +current_key+ + # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ + # +current_key+ main model that will be a key in the hash + # +value+ existing model to be included in the hash + def add_model_value(current_key, value) + @attributes_finder.parse(value) { |hash| value = { value => hash } } + old_values = @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten + end + + # Construct a new hash or merge with an existing one a model configuration + # This is to fulfil +to_json+ requirements. + # +value+ existing model to be included in the hash + # +hash+ hash containing configuration generated mainly from +@attributes_finder+ + def hash_or_merge(value, hash) + value.is_a?(Hash) ? value.merge(hash) : { value => hash } + end + end + end +end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb new file mode 100644 index 00000000000..b872780f20a --- /dev/null +++ b/lib/gitlab/import_export/relation_factory.rb @@ -0,0 +1,128 @@ +module Gitlab + module ImportExport + class RelationFactory + + OVERRIDES = { snippets: :project_snippets, + pipelines: 'Ci::Pipeline', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + + def self.create(*args) + new(*args).create + end + + def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + @relation_name = OVERRIDES[relation_sym] || relation_sym + @relation_hash = relation_hash.except('id', 'noteable_id') + @members_mapper = members_mapper + @user = user + end + + # Creates an object from an actual model with name "relation_sym" with params from + # the relation_hash, updating references with new object IDs, mapping users using + # the "members_mapper" object, also updating notes if required. + def create + set_note_author if @relation_name == :notes + update_user_references + update_project_references + reset_ci_tokens if @relation_name == 'Ci::Trigger' + + generate_imported_object + end + + private + + def update_user_references + USER_REFERENCES.each do |reference| + if @relation_hash[reference] + @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] + end + end + end + + # Sets the author for a note. If the user importing the project + # has admin access, an actual mapping with new project members + # will be used. Otherwise, a note stating the original author name + # is left. + def set_note_author + old_author_id = @relation_hash['author_id'] + + # Users with admin access can map users + @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id + + author = @relation_hash.delete('author') + + update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + end + + def missing_author?(old_author_id) + !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + end + + def missing_author_note(updated_at, author_name) + timestamp = updated_at.split('.').first + "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" + end + + def generate_imported_object + if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes + trace = @relation_hash.delete('trace') + imported_object do |object| + object.trace = trace + object.commit_id = nil + end + else + imported_object + end + end + + def update_project_references + project_id = @relation_hash.delete('project_id') + + # project_id may not be part of the export, but we always need to populate it if required. + @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id') + @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] + @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id'] + + # If source and target are the same, populate them with the new project ID. + if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] && + @relation_hash['target_project_id'] == @relation_hash['source_project_id'] + @relation_hash['source_project_id'] = project_id + end + end + + def reset_ci_tokens + return unless Gitlab::ImportExport.reset_tokens? + + # If we import/export a project to the same instance, tokens will have to be reset. + @relation_hash['token'] = nil + end + + def relation_class + @relation_class ||= @relation_name.to_s.classify.constantize + end + + def imported_object + imported_object = relation_class.new(@relation_hash) + yield(imported_object) if block_given? + imported_object.importing = true if imported_object.respond_to?(:importing) + imported_object + end + + def update_note_for_missing_author(author_name) + @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? + @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name) + end + + def admin_user? + @user.is_admin? + end + end + end +end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb new file mode 100644 index 00000000000..546dae4d122 --- /dev/null +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -0,0 +1,39 @@ +module Gitlab + module ImportExport + class RepoRestorer + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:, path_to_bundle:, wiki: false) + @project = project + @path_to_bundle = path_to_bundle + @shared = shared + @wiki = wiki + end + + def restore + return wiki? unless File.exist?(@path_to_bundle) + + FileUtils.mkdir_p(path_to_repo) + + git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) + rescue => e + @shared.error(e) + false + end + + private + + def repos_path + Gitlab.config.gitlab_shell.repos_path + end + + def path_to_repo + @project.repository.path_to_repo + end + + def wiki? + @wiki + end + end + end +end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb new file mode 100644 index 00000000000..cce43fe994b --- /dev/null +++ b/lib/gitlab/import_export/repo_saver.rb @@ -0,0 +1,35 @@ +module Gitlab + module ImportExport + class RepoSaver + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :full_path + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return false if @project.empty_repo? + + @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename) + bundle_to_disk + end + + private + + def bundle_to_disk + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: @full_path) + rescue => e + @shared.error(e) + false + end + + def path_to_repo + @project.repository.path_to_repo + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb new file mode 100644 index 00000000000..f38229c6c59 --- /dev/null +++ b/lib/gitlab/import_export/saver.rb @@ -0,0 +1,42 @@ +module Gitlab + module ImportExport + class Saver + include Gitlab::ImportExport::CommandLineUtil + + def self.save(*args) + new(*args).save + end + + def initialize(shared:) + @shared = shared + end + + def save + if compress_and_save + remove_export_path + Rails.logger.info("Saved project export #{archive_file}") + archive_file + else + false + end + rescue => e + @shared.error(e) + false + end + + private + + def compress_and_save + tar_czf(archive: archive_file, dir: @shared.export_path) + end + + def remove_export_path + FileUtils.rm_rf(@shared.export_path) + end + + def archive_file + @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz") + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb new file mode 100644 index 00000000000..6aff05b886a --- /dev/null +++ b/lib/gitlab/import_export/shared.rb @@ -0,0 +1,30 @@ +module Gitlab + module ImportExport + class Shared + + attr_reader :errors, :opts + + def initialize(opts) + @opts = opts + @errors = [] + end + + def export_path + @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path]) + end + + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) + end + + private + + def error_out(message, caller) + Rails.logger.error("Import/Export error raised on #{caller}: #{message}") + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb new file mode 100644 index 00000000000..df19354b76e --- /dev/null +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -0,0 +1,14 @@ +module Gitlab + module ImportExport + class UploadsRestorer < UploadsSaver + def restore + return true unless File.directory?(uploads_export_path) + + copy_files(uploads_export_path, uploads_path) + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb new file mode 100644 index 00000000000..7292e9d9712 --- /dev/null +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class UploadsSaver + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + return true unless File.directory?(uploads_path) + + copy_files(uploads_path, uploads_export_path) + rescue => e + @shared.error(e) + false + end + + private + + def copy_files(source, destination) + FileUtils.mkdir_p(destination) + FileUtils.copy_entry(source, destination) + true + end + + def uploads_export_path + File.join(@shared.export_path, 'uploads') + end + + def uploads_path + File.join(Rails.root.join('public/uploads'), @project.path_with_namespace) + end + end + end +end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb new file mode 100644 index 00000000000..cf5c62c5e3c --- /dev/null +++ b/lib/gitlab/import_export/version_checker.rb @@ -0,0 +1,36 @@ +module Gitlab + module ImportExport + class VersionChecker + + def self.check!(*args) + new(*args).check! + end + + def initialize(shared:) + @shared = shared + end + + def check! + version = File.open(version_file, &:readline) + verify_version!(version) + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + + def verify_version!(version) + if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) + raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + else + true + end + end + end + end +end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb new file mode 100644 index 00000000000..f7f73dc9343 --- /dev/null +++ b/lib/gitlab/import_export/version_saver.rb @@ -0,0 +1,25 @@ +module Gitlab + module ImportExport + class VersionSaver + + def initialize(shared:) + @shared = shared + end + + def save + FileUtils.mkdir_p(@shared.export_path) + + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + rescue => e + @shared.error(e) + false + end + + private + + def version_file + File.join(@shared.export_path, Gitlab::ImportExport.version_filename) + end + end + end +end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb new file mode 100644 index 00000000000..1eedae39f8a --- /dev/null +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -0,0 +1,33 @@ +module Gitlab + module ImportExport + class WikiRepoSaver < RepoSaver + def save + @wiki = ProjectWiki.new(@project) + return true unless wiki_repository_exists? # it's okay to have no Wiki + bundle_to_disk(File.join(@shared.export_path, project_filename)) + end + + def bundle_to_disk(full_path) + FileUtils.mkdir_p(@shared.export_path) + git_bundle(repo_path: path_to_repo, bundle_path: full_path) + rescue => e + @shared.error(e) + false + end + + private + + def project_filename + "project.wiki.bundle" + end + + def path_to_repo + @wiki.repository.path_to_repo + end + + def wiki_repository_exists? + File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty? + end + end + end +end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index ccfdfbe73e8..948d43582cf 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,7 +20,8 @@ module Gitlab 'Gitorious.org' => 'gitorious', 'Google Code' => 'google_code', 'FogBugz' => 'fogbugz', - 'Any repo by URL' => 'git', + 'Repo by URL' => 'git', + 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index d81d26754fe..dcec7543c13 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -148,23 +148,8 @@ module Gitlab proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) - trans = Gitlab::Metrics::Instrumentation.transaction - - if trans - start = Time.now - cpu_start = Gitlab::Metrics::System.cpu_time - retval = super - duration = (Time.now - start) * 1000.0 - - if duration >= Gitlab::Metrics.method_call_threshold - cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start - - trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration, cpu_duration: cpu_duration }, - method: #{label.inspect}) - end - - retval + if trans = Gitlab::Metrics::Instrumentation.transaction + trans.measure_method(#{label.inspect}) { super } else super end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb new file mode 100644 index 00000000000..faf0d9b6318 --- /dev/null +++ b/lib/gitlab/metrics/method_call.rb @@ -0,0 +1,52 @@ +module Gitlab + module Metrics + # Class for tracking timing information about method calls + class MethodCall + attr_reader :real_time, :cpu_time, :call_count + + # name - The full name of the method (including namespace) such as + # `User#sign_in`. + # + # series - The series to use for storing the data. + def initialize(name, series) + @name = name + @series = series + @real_time = 0.0 + @cpu_time = 0.0 + @call_count = 0 + end + + # Measures the real and CPU execution time of the supplied block. + def measure + start_real = Time.now + start_cpu = System.cpu_time + retval = yield + + @real_time += (Time.now - start_real) * 1000.0 + @cpu_time += System.cpu_time.to_f - start_cpu + @call_count += 1 + + retval + end + + # Returns a Metric instance of the current method call. + def to_metric + Metric.new( + @series, + { + duration: real_time, + cpu_duration: cpu_time, + call_count: call_count + }, + method: @name + ) + end + + # Returns true if the total runtime of this method exceeds the method call + # threshold. + def above_threshold? + real_time >= Metrics.method_call_threshold + end + end + end +end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 3fe27779d03..e61670f491c 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -35,7 +35,7 @@ module Gitlab def transaction_from_env(env) trans = Transaction.new - trans.set(:request_uri, env['REQUEST_URI']) + trans.set(:request_uri, filtered_path(env)) trans.set(:request_method, env['REQUEST_METHOD']) trans @@ -54,6 +54,10 @@ module Gitlab private + def filtered_path(env) + ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI'] + end + def endpoint_paths_cache @endpoint_paths_cache ||= Hash.new do |hash, http_method| hash[http_method] = Hash.new do |inner_hash, raw_path| diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 2578ddc49f4..4bc5081aa03 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,7 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values + attr_reader :tags, :values, :methods attr_accessor :action @@ -16,6 +16,7 @@ module Gitlab # plus method name. def initialize(action = nil) @metrics = [] + @methods = {} @started_at = nil @finished_at = nil @@ -51,9 +52,23 @@ module Gitlab end def add_metric(series, values, tags = {}) - prefix = sidekiq? ? 'sidekiq_' : 'rails_' + @metrics << Metric.new("#{series_prefix}#{series}", values, tags) + end + + # Measures the time it takes to execute a method. + # + # Multiple calls to the same method add up to the total runtime of the + # method. + # + # name - The full name of the method to measure (e.g. `User#sign_in`). + def measure_method(name, &block) + unless @methods[name] + series = "#{series_prefix}#{Instrumentation::SERIES}" + + @methods[name] = MethodCall.new(name, series) + end - @metrics << Metric.new("#{prefix}#{series}", values, tags) + @methods[name].measure(&block) end def increment(name, value) @@ -84,7 +99,13 @@ module Gitlab end def submit - metrics = @metrics.map do |metric| + submit = @metrics.dup + + @methods.each do |name, method| + submit << method.to_metric if method.above_threshold? + end + + submit_hashes = submit.map do |metric| hash = metric.to_hash hash[:tags][:action] ||= @action if @action @@ -92,12 +113,16 @@ module Gitlab hash end - Metrics.submit_metrics(metrics) + Metrics.submit_metrics(submit_hashes) end def sidekiq? Sidekiq.server? end + + def series_prefix + sidekiq? ? 'sidekiq_' : 'rails_' + end end end end |