diff options
Diffstat (limited to 'lib')
44 files changed, 936 insertions, 285 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4be7707d3e7..f858d9fa23d 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -62,6 +62,14 @@ module API expose :admin?, as: :is_admin end + class UserStatus < Grape::Entity + expose :emoji + expose :message + expose :message_html do |entity| + MarkupHelper.markdown_field(entity, :message) + end + end + class Email < Grape::Entity expose :id, :email end diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb index cde51fccc62..c9c44e3c218 100644 --- a/lib/api/helpers/headers_helpers.rb +++ b/lib/api/helpers/headers_helpers.rb @@ -3,7 +3,11 @@ module API module HeadersHelpers def set_http_headers(header_data) header_data.each do |key, value| - header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value + if value.is_a?(Enumerable) + raise ArgumentError.new("Header value should be a string") + end + + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value.to_s end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index a9803be9f69..516f25db15b 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -11,7 +11,8 @@ module API # # Params: # key_id - ssh key id for Git over SSH - # user_id - user id for Git over HTTP + # user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode + # username - user name for Git over SSH in keyless SSH cert mode # protocol - Git access protocol being used, e.g. HTTP or SSH # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) @@ -28,6 +29,8 @@ module API Key.find_by(id: params[:key_id]) elsif params[:user_id] User.find_by(id: params[:user_id]) + elsif params[:username] + User.find_by_username(params[:username]) end protocol = params[:protocol] @@ -58,6 +61,7 @@ module API { status: true, gl_repository: gl_repository, + gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, # This repository_path is a bogus value but gitlab-shell still requires @@ -71,10 +75,17 @@ module API post "/lfs_authenticate" do status 200 - key = Key.find(params[:key_id]) - key.update_last_used_at + if params[:key_id] + actor = Key.find(params[:key_id]) + actor.update_last_used_at + elsif params[:user_id] + actor = User.find_by(id: params[:user_id]) + raise ActiveRecord::RecordNotFound.new("No such user id!") unless actor + else + raise ActiveRecord::RecordNotFound.new("No key_id or user_id passed!") + end - token_handler = Gitlab::LfsToken.new(key) + token_handler = Gitlab::LfsToken.new(actor) { username: token_handler.actor_name, @@ -100,7 +111,7 @@ module API end # - # Discover user by ssh key or user id + # Discover user by ssh key, user id or username # get "/discover" do if params[:key_id] @@ -108,6 +119,8 @@ module API user = key.user elsif params[:user_id] user = User.find_by(id: params[:user_id]) + elsif params[:username] + user = User.find_by(username: params[:username]) end present user, with: Entities::UserSafe @@ -141,22 +154,30 @@ module API post '/two_factor_recovery_codes' do status 200 - key = Key.find_by(id: params[:key_id]) + if params[:key_id] + key = Key.find_by(id: params[:key_id]) - if key - key.update_last_used_at - else - break { 'success' => false, 'message' => 'Could not find the given key' } - end + if key + key.update_last_used_at + else + break { 'success' => false, 'message' => 'Could not find the given key' } + end - if key.is_a?(DeployKey) - break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } - end + if key.is_a?(DeployKey) + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user - user = key.user + unless user + break { success: false, message: 'Could not find a user for the given key' } + end + elsif params[:user_id] + user = User.find_by(id: params[:user_id]) - unless user - break { success: false, message: 'Could not find a user for the given key' } + unless user + break { success: false, message: 'Could not find the given user' } + end end unless user.two_factor_enabled? diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 25185d6edc8..bda05d1795b 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -162,6 +162,9 @@ module API desc: 'The IID of a merge request for which to resolve discussions' optional :discussion_to_resolve, type: String, desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`' + optional :iid, type: Integer, + desc: 'The internal ID of a project issue. Available only for admins and project owners.' + use :issue_params end post ':id/issues' do @@ -169,9 +172,10 @@ module API authorize! :create_issue, user_project - # Setting created_at time only allowed for admins and project owners + # Setting created_at time or iid only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) + params.delete(:iid) end issue_params = declared_params(include_missing: false) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 2621c9f8fc2..abad418771c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -380,7 +380,7 @@ module API end get ':id/merge_requests/:merge_request_iid/closes_issues' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user)) issues = paginate(issues) external_issues, internal_issues = issues.partition { |issue| issue.is_a?(ExternalIssue) } diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 7adde79d6c3..5738bf220c6 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -321,7 +321,7 @@ module API post ':id/archive' do authorize!(:archive_project, user_project) - user_project.archive! + ::Projects::UpdateService.new(user_project, current_user, archived: true).execute present user_project, with: Entities::Project end @@ -332,7 +332,7 @@ module API post ':id/unarchive' do authorize!(:archive_project, user_project) - user_project.unarchive! + ::Projects::UpdateService.new(@project, current_user, archived: false).execute present user_project, with: Entities::Project end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index c7595493e11..c9931c2d603 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -80,7 +80,15 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) optional :last_update, type: String, desc: %q(Runner's queue last_update token) - optional :info, type: Hash, desc: %q(Runner's metadata) + optional :info, type: Hash, desc: %q(Runner's metadata) do + optional :name, type: String, desc: %q(Runner's name) + optional :version, type: String, desc: %q(Runner's version) + optional :revision, type: String, desc: %q(Runner's revision) + optional :platform, type: String, desc: %q(Runner's platform) + optional :architecture, type: String, desc: %q(Runner's architecture) + optional :executor, type: String, desc: %q(Runner's executor) + optional :features, type: Hash, desc: %q(Runner's features) + end optional :session, type: Hash, desc: %q(Runner's session data) do optional :url, type: String, desc: %q(Session's url) optional :certificate, type: String, desc: %q(Session's certificate) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index be7c7f58d51..897010217dc 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -62,7 +62,7 @@ module API requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." end optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' - optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project manifest], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' optional :koding_enabled, type: Boolean, desc: 'Enable Koding' given koding_enabled: ->(val) { val } do @@ -76,8 +76,8 @@ module API requires :metrics_host, type: String, desc: 'The InfluxDB host' requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' - requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' end @@ -127,9 +127,7 @@ module API optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' - optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' - optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' - optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' + optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/users.rb b/lib/api/users.rb index e83887b3e9e..b0811bb4aad 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -121,6 +121,17 @@ module API present user, opts end + desc "Get the status of a user" + params do + requires :id_or_username, type: String, desc: 'The ID or username of the user' + end + get ":id_or_username/status" do + user = find_user(params[:id_or_username]) + not_found!('User') unless user && can?(current_user, :read_user, user) + + present user.status || {}, with: Entities::UserStatus + end + desc 'Create a user. Available only for admins.' do success Entities::UserPublic end @@ -740,6 +751,30 @@ module API present paginate(activities), with: Entities::UserActivity end + + desc 'Set the status of the current user' do + success Entities::UserStatus + end + params do + optional :emoji, type: String, desc: "The emoji to set on the status" + optional :message, type: String, desc: "The status message to set" + end + put "status" do + forbidden! unless can?(current_user, :update_user_status, current_user) + + if ::Users::SetStatusService.new(current_user, declared_params).execute + present current_user.status, with: Entities::UserStatus + else + render_validation_error!(current_user.status) + end + end + + desc 'get the status of the current user' do + success Entities::UserStatus + end + get 'status' do + present current_user.status || {}, with: Entities::UserStatus + end end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index af762db517c..906ed498026 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -1,10 +1,7 @@ require 'yaml' -require_relative 'helper' module Backup class Repository - include Backup::Helper - attr_reader :progress def initialize(progress) @@ -42,131 +39,36 @@ module Backup end def prepare_directories - Gitlab.config.repositories.storages.each do |name, repository_storage| - delete_all_repositories(name, repository_storage) + Gitlab.config.repositories.storages.each do |name, _repository_storage| + Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories end end def backup_project(project) - gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - backup_project_gitaly(project) - else - backup_project_local(project) - end - end - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_project_gitaly(project) path_to_project_bundle = path_to_bundle(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .create_bundle(path_to_project_bundle) - end - - def backup_project_local(project) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - - path_to_project_bundle = path_to_bundle(project) - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - progress_warn(project, cmd.join(' '), output) unless status.zero? - end - - def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories - else - local_delete_all_repositories(name, repository_storage) - end - end - end - - def local_delete_all_repositories(name, repository_storage) - path = repository_storage.legacy_disk_path - return unless File.exist?(path) - - bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) - FileUtils.mkdir_p(bk_repos_path, mode: 0700) - files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] - - begin - FileUtils.mv(files, bk_repos_path) - rescue Errno::EACCES - access_denied_error(path) - rescue Errno::EBUSY - resource_busy_error(path) - end - end - def local_restore_custom_hooks(project, dir) - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - def gitaly_restore_custom_hooks(project, dir) - custom_hooks_path = path_to_tars(project, dir) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') end - def local_backup_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - break unless File.exist?(File.join(path_to_project_repo, dir)) - - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - end + def backup_custom_hooks(project) + FileUtils.mkdir_p(project_backup_path(project)) - def gitaly_backup_custom_hooks(project) - FileUtils.mkdir_p(path_to_tars(project)) - custom_hooks_path = path_to_tars(project, 'custom_hooks') + custom_hooks_path = custom_hooks_tar(project) Gitlab::GitalyClient::RepositoryService.new(project.repository) .backup_custom_hooks(custom_hooks_path) end - def backup_custom_hooks(project) - gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_backup_custom_hooks(project) - else - local_backup_custom_hooks(project) - end - end - end - def restore_custom_hooks(project) - in_path(path_to_tars(project)) do |dir| - gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_restore_custom_hooks(project, dir) - else - local_restore_custom_hooks(project, dir) - end - end - end + return unless Dir.exist?(project_backup_path(project)) + return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? + + custom_hooks_path = custom_hooks_tar(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .restore_custom_hooks(custom_hooks_path) end def restore @@ -181,7 +83,8 @@ module Backup restore_repo_success = nil if File.exist?(path_to_project_bundle) begin - project.repository.create_from_bundle path_to_project_bundle + project.repository.create_from_bundle(path_to_project_bundle) + restore_custom_hooks(project) restore_repo_success = true rescue => e restore_repo_success = false @@ -197,8 +100,6 @@ module Backup progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) end - restore_custom_hooks(project) - wiki = ProjectWiki.new(project) path_to_wiki_bundle = path_to_bundle(wiki) @@ -219,48 +120,28 @@ module Backup protected - def path_to_repo(project) - project.repository.path_to_repo - end - def path_to_bundle(project) File.join(backup_repos_path, project.disk_path + '.bundle') end - def path_to_tars(project, dir = nil) - path = File.join(backup_repos_path, project.disk_path) + def project_backup_path(project) + File.join(backup_repos_path, project.disk_path) + end - if dir - File.join(path, "#{dir}.tar") - else - path - end + def custom_hooks_tar(project) + File.join(project_backup_path(project), "custom_hooks.tar") end def backup_repos_path File.join(Gitlab.config.backup.path, 'repositories') end - def in_path(path) - return unless Dir.exist?(path) - - dir_entries = Dir.entries(path) - - if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') - yield('custom_hooks') - end - end - def prepare FileUtils.rm_rf(backup_repos_path) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir(backup_repos_path, mode: 0700) end - def silent - { err: '/dev/null', out: '/dev/null' } - end - private def progress_warn(project, cmd, output) @@ -273,18 +154,8 @@ module Backup project_or_wiki.repository.empty? end - def repository_storage_paths_args - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - def display_repo_path(project) project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path end - - def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound, GRPC::BadStatus => e - raise Error, e - end end end diff --git a/lib/banzai/pipeline/emoji_pipeline.rb b/lib/banzai/pipeline/emoji_pipeline.rb new file mode 100644 index 00000000000..a1b522f4303 --- /dev/null +++ b/lib/banzai/pipeline/emoji_pipeline.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class EmojiPipeline < BasePipeline + # These filters will only perform sanitization of the content, preventing + # XSS, and replace emoji. + def self.filters + @filters ||= FilterArray[ + Filter::HtmlEntityFilter, + Filter::SanitizationFilter, + Filter::EmojiFilter + ] + end + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb index d27b2b0e72f..09c5ef3ad94 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -46,6 +46,10 @@ class Feature get(key).enabled?(thing) end + def disabled?(key, thing = nil) + !enabled?(key, thing) + end + def enable(key, thing = true) get(key).enable(thing) end diff --git a/lib/gitlab/auth/activity.rb b/lib/gitlab/auth/activity.rb index 9f84c578d4f..761f0819c60 100644 --- a/lib/gitlab/auth/activity.rb +++ b/lib/gitlab/auth/activity.rb @@ -18,8 +18,7 @@ module Gitlab user_blocked: 'Counter of sign in attempts when user is blocked' }.freeze - def initialize(user, opts) - @user = user + def initialize(opts) @opts = opts end @@ -32,8 +31,6 @@ module Gitlab when :invalid self.class.user_password_invalid_counter_increment! end - - self.class.user_blocked_counter_increment! if @user&.blocked? end def user_authenticated! @@ -51,6 +48,10 @@ module Gitlab end end + def user_blocked! + self.class.user_blocked_counter_increment! + end + def user_session_destroyed! self.class.user_session_destroyed_counter_increment! end diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb index b6d2adc834b..50712d7eac2 100644 --- a/lib/gitlab/auth/blocked_user_tracker.rb +++ b/lib/gitlab/auth/blocked_user_tracker.rb @@ -2,58 +2,21 @@ module Gitlab module Auth class BlockedUserTracker - include Gitlab::Utils::StrongMemoize - - ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters' - - def initialize(env) - @env = env - end - - def user_blocked? - user&.blocked? + def initialize(user, auth) + @user = user + @auth = auth end - def user - return unless has_user_blocked_message? + def log_activity! + return unless @user.blocked? - strong_memoize(:user) do - # Check for either LDAP or regular GitLab account logins - login = @env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'username') || - @env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login') + Gitlab::AppLogger.info <<~INFO + "Failed login for blocked user: user=#{@user.username} ip=#{@auth.request.ip}") + INFO - User.by_login(login) if login.present? - end + SystemHooksService.new.execute_hooks_for(@user, :failed_login) rescue TypeError end - - def log_blocked_user_activity! - return unless user_blocked? - - Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{@env['REMOTE_ADDR']}") - SystemHooksService.new.execute_hooks_for(user, :failed_login) - true - rescue TypeError - end - - private - - ## - # Devise calls User#active_for_authentication? on the User model and then - # throws an exception to Warden with User#inactive_message: - # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8 - # - # Since Warden doesn't pass the user record to the failure handler, we - # need to do a database lookup with the username. We can limit the - # lookups to happen when the user was blocked by checking the inactive - # message passed along by Warden. - # - def has_user_blocked_message? - strong_memoize(:user_blocked_message) do - message = @env.dig('warden.options', :message) - message == User::BLOCKED_MESSAGE - end - end end end end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d3f66877672..36c85dec544 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -46,7 +46,11 @@ module Gitlab # arguments - The arguments to pass to the background migration's "perform" # method. def self.perform(class_name, arguments) - const_get(class_name).new.perform(*arguments) + migration_class_for(class_name).new.perform(*arguments) + end + + def self.migration_class_for(class_name) + const_get(class_name) end end end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 4ca5a78e068..04aa6aab771 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -26,6 +26,12 @@ module Gitlab end end + # This is called from within a rake task only used by Admins, so allow writing + # to STDOUT + def self.log(message) + puts message # rubocop:disable Rails/Output + end + attr_reader :user, :project_name, :bare_repo delegate :log, to: :class @@ -59,11 +65,10 @@ module Gitlab import_type: 'bare_repository', namespace_id: group&.id).execute - if project.persisted? && mv_repo(project) + if project.persisted? && mv_repositories(project) log " * Created #{project.name} (#{project_full_path})".color(:green) project.write_repository_config - Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path) ProjectCacheWorker.perform_async(project.id) else @@ -74,12 +79,11 @@ module Gitlab project end - def mv_repo(project) - storage_path = storage_path_for_shard(project.repository_storage) - FileUtils.mv(repo_path, project.repository.path_to_repo) + def mv_repositories(project) + mv_repo(bare_repo.repo_path, project.repository) if bare_repo.wiki_exists? - FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git')) + mv_repo(bare_repo.wiki_path, project.wiki.repository) end true @@ -89,6 +93,11 @@ module Gitlab false end + def mv_repo(path, repository) + repository.create_from_bundle(bundle(path)) + FileUtils.rm_rf(path) + end + def storage_path_for_shard(shard) Gitlab.config.repositories.storages[shard].legacy_disk_path end @@ -101,10 +110,17 @@ module Gitlab Groups::NestedCreateService.new(user, group_path: group_path).execute end - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - def self.log(message) - puts message # rubocop:disable Rails/Output + def bundle(repo_path) + # TODO: we could save some time and disk space by using + # `git bundle create - --all` and streaming the bundle directly to + # Gitaly, rather than writing it on disk first + bundle_path = "#{repo_path}.bundle" + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) + output, status = Gitlab::Popen.popen(cmd) + + raise output unless status.zero? + + bundle_path end end end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fe267248275..c0c666dfb7b 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -1,5 +1,3 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953 -# module Gitlab module BareRepositoryImport class Repository diff --git a/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb b/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb new file mode 100644 index 00000000000..65f65cdce08 --- /dev/null +++ b/lib/gitlab/ci/build/artifacts/gzip_file_adapter.rb @@ -0,0 +1,46 @@ +module Gitlab + module Ci + module Build + module Artifacts + class GzipFileAdapter + attr_reader :stream + + InvalidStreamError = Class.new(StandardError) + + def initialize(stream) + raise InvalidStreamError, "Stream is required" unless stream + + @stream = stream + end + + def each_blob + stream.seek(0) + + until stream.eof? + gzip(stream) do |gz| + yield gz.read, gz.orig_name + unused = gz.unused&.length.to_i + # pos has already reached to EOF at the moment + # We rewind the pos to the top of unused files + # to read next gzip stream, to support multistream archives + # https://golang.org/src/compress/gzip/gunzip.go#L117 + stream.seek(-unused, IO::SEEK_CUR) + end + end + end + + private + + def gzip(stream, &block) + gz = Zlib::GzipReader.new(stream) + yield(gz) + rescue Zlib::Error => e + raise InvalidStreamError, e.message + ensure + gz&.finish + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb new file mode 100644 index 00000000000..a4eccc08dfc --- /dev/null +++ b/lib/gitlab/ci/parsers.rb @@ -0,0 +1,9 @@ +module Gitlab + module Ci + module Parsers + def self.fabricate!(file_type) + "Gitlab::Ci::Parsers::#{file_type.classify}".constantize.new + end + end + end +end diff --git a/lib/gitlab/ci/parsers/junit.rb b/lib/gitlab/ci/parsers/junit.rb new file mode 100644 index 00000000000..3c4668ec13b --- /dev/null +++ b/lib/gitlab/ci/parsers/junit.rb @@ -0,0 +1,69 @@ +module Gitlab + module Ci + module Parsers + class Junit + attr_reader :data + + JunitParserError = Class.new(StandardError) + + def parse!(xml_data, test_suite) + @data = Hash.from_xml(xml_data) + + each_suite do |testcases| + testcases.each do |testcase| + test_case = create_test_case(testcase) + test_suite.add_test_case(test_case) + end + end + rescue REXML::ParseException => e + raise JunitParserError, "XML parsing failed: #{e.message}" + rescue => e + raise JunitParserError, "JUnit parsing failed: #{e.message}" + end + + private + + def each_suite + testsuites.each do |testsuite| + yield testcases(testsuite) + end + end + + def testsuites + if data['testsuites'] + data['testsuites']['testsuite'] + else + [data['testsuite']] + end + end + + def testcases(testsuite) + if testsuite['testcase'].is_a?(Array) + testsuite['testcase'] + else + [testsuite['testcase']] + end + end + + def create_test_case(data) + if data['failure'] + status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED + system_output = data['failure'] + else + status = ::Gitlab::Ci::Reports::TestCase::STATUS_SUCCESS + system_output = nil + end + + ::Gitlab::Ci::Reports::TestCase.new( + classname: data['classname'], + name: data['name'], + file: data['file'], + execution_time: data['time'], + status: status, + system_output: system_output + ) + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb new file mode 100644 index 00000000000..b4d08ed257f --- /dev/null +++ b/lib/gitlab/ci/reports/test_case.rb @@ -0,0 +1,32 @@ +module Gitlab + module Ci + module Reports + class TestCase + STATUS_SUCCESS = 'success'.freeze + STATUS_FAILED = 'failed'.freeze + STATUS_SKIPPED = 'skipped'.freeze + STATUS_ERROR = 'error'.freeze + STATUS_TYPES = [STATUS_SUCCESS, STATUS_FAILED, STATUS_SKIPPED, STATUS_ERROR].freeze + + attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key + + def initialize(name:, classname:, execution_time:, status:, file: nil, system_output: nil, stack_trace: nil) + @name = name + @classname = classname + @file = file + @execution_time = execution_time.to_f + @status = status + @system_output = system_output + @stack_trace = stack_trace + @key = sanitize_key_name("#{classname}_#{name}") + end + + private + + def sanitize_key_name(key) + key.gsub(/[^0-9A-Za-z]/, '-') + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb new file mode 100644 index 00000000000..c6e732e68eb --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + module Reports + class TestReports + attr_reader :test_suites + + def initialize + @test_suites = {} + end + + def get_suite(suite_name) + test_suites[suite_name] ||= TestSuite.new(suite_name) + end + + def total_time + test_suites.values.sum(&:total_time) + end + + def total_count + test_suites.values.sum(&:total_count) + end + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}_count") do + test_suites.values.sum { |suite| suite.public_send("#{status_type}_count") } # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb new file mode 100644 index 00000000000..c0943f5a51a --- /dev/null +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + module Reports + class TestReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_reports, :head_reports + + def initialize(base_reports, head_reports) + @base_reports = base_reports || TestReports.new + @head_reports = head_reports + end + + def suite_comparers + strong_memoize(:suite_comparers) do + head_reports.test_suites.map do |name, test_suite| + TestSuiteComparer.new(name, base_reports.get_suite(name), test_suite) + end + end + end + + def total_status + if suite_comparers.any? { |suite| suite.total_status == TestCase::STATUS_FAILED } + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + %w(total_count resolved_count failed_count).each do |method| + define_method(method) do + suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb new file mode 100644 index 00000000000..b722d0ba735 --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -0,0 +1,54 @@ +module Gitlab + module Ci + module Reports + class TestSuite + attr_reader :name + attr_reader :test_cases + attr_reader :total_time + + def initialize(name = nil) + @name = name + @test_cases = {} + @total_time = 0.0 + @duplicate_cases = [] + end + + def add_test_case(test_case) + @duplicate_cases << test_case if existing_key?(test_case) + + @test_cases[test_case.status] ||= {} + @test_cases[test_case.status][test_case.key] = test_case + @total_time += test_case.execution_time + end + + def total_count + test_cases.values.sum(&:count) + end + + def total_status + if failed_count > 0 || error_count > 0 + TestCase::STATUS_FAILED + else + TestCase::STATUS_SUCCESS + end + end + + TestCase::STATUS_TYPES.each do |status_type| + define_method("#{status_type}") do + test_cases[status_type] || {} + end + + define_method("#{status_type}_count") do + test_cases[status_type]&.length.to_i + end + end + + private + + def existing_key?(test_case) + @test_cases[test_case.status]&.key?(test_case.key) + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb new file mode 100644 index 00000000000..642aa593092 --- /dev/null +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -0,0 +1,57 @@ +module Gitlab + module Ci + module Reports + class TestSuiteComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :name, :base_suite, :head_suite + + def initialize(name, base_suite, head_suite) + @name = name + @base_suite = base_suite || TestSuite.new + @head_suite = head_suite + end + + def new_failures + strong_memoize(:new_failures) do + head_suite.failed.reject do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def existing_failures + strong_memoize(:existing_failures) do + head_suite.failed.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def resolved_failures + strong_memoize(:resolved_failures) do + head_suite.success.select do |key, _| + base_suite.failed.include?(key) + end.values + end + end + + def total_count + head_suite.total_count + end + + def total_status + head_suite.total_status + end + + def resolved_count + resolved_failures.count + end + + def failed_count + new_failures.count + existing_failures.count + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 155f4fc1343..703f0b9217b 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -4,12 +4,13 @@ module Gitlab module Build class Failed < Status::Extended REASONS = { - 'unknown_failure' => 'unknown failure', - 'script_failure' => 'script failure', - 'api_failure' => 'API failure', - 'stuck_or_timeout_failure' => 'stuck or timeout failure', - 'runner_system_failure' => 'runner system failure', - 'missing_dependency_failure' => 'missing dependency failure' + unknown_failure: 'unknown failure', + script_failure: 'script failure', + api_failure: 'API failure', + stuck_or_timeout_failure: 'stuck or timeout failure', + runner_system_failure: 'runner system failure', + missing_dependency_failure: 'missing dependency failure', + runner_unsupported: 'unsupported runner' }.freeze def status_tooltip @@ -31,7 +32,11 @@ module Gitlab end def description - "<br> (#{REASONS[subject.failure_reason]})" + "<br> (#{failure_reason_message})" + end + + def failure_reason_message + REASONS.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/cleanup/project_upload_file_finder.rb b/lib/gitlab/cleanup/project_upload_file_finder.rb new file mode 100644 index 00000000000..2ee8b60e76a --- /dev/null +++ b/lib/gitlab/cleanup/project_upload_file_finder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploadFileFinder + FIND_BATCH_SIZE = 500 + ABSOLUTE_UPLOAD_DIR = FileUploader.root.freeze + EXCLUDED_SYSTEM_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/-/*".freeze + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + + # Paths are relative to the upload directory + def each_file_batch(batch_size: FIND_BATCH_SIZE, &block) + cmd = build_find_command(ABSOLUTE_UPLOAD_DIR) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + private + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) if paths.any? + end + + def build_find_command(search_dir) + cmd = %W[find -L #{search_dir} + -type f + ! ( -path #{EXCLUDED_SYSTEM_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] + + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice + + log_msg = "find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg + + cmd + end + + def which_ionice + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, + # even though it isn't very nice + false + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb new file mode 100644 index 00000000000..f55ab535efe --- /dev/null +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class ProjectUploads + LOST_AND_FOUND = File.join(ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR, '-', 'project-lost-found') + + attr_reader :logger + + def initialize(logger: nil) + @logger = logger || Rails.logger + end + + def run!(dry_run: true) + logger.info "Looking for orphaned project uploads to clean up#{'. Dry run' if dry_run}..." + + each_orphan_file do |path, upload_path| + result = cleanup(path, upload_path, dry_run) + + logger.info result + end + end + + private + + def cleanup(path, upload_path, dry_run) + # This happened in staging: + # `find` returned a path on which `File.delete` raised `Errno::ENOENT` + return "Cannot find file: #{path}" unless File.exist?(path) + + correct_path = upload_path && find_correct_path(upload_path) + + if correct_path + move(path, correct_path, 'fix', dry_run) + else + move_to_lost_and_found(path, dry_run) + end + end + + # Accepts a path in the form of "#{hex_secret}/#{filename}" + def find_correct_path(upload_path) + upload = Upload.find_by(uploader: 'FileUploader', path: upload_path) + return unless upload && upload.local? && upload.model + + upload.absolute_path + rescue => e + logger.error e.message + + # absolute_path depends on a lot of code. If it doesn't work, then it + # it doesn't matter if the upload file is in the right place. Treat it + # as uncorrectable. + # I.e. the project record might be missing, which raises an exception. + nil + end + + def move_to_lost_and_found(path, dry_run) + new_path = path.sub(/\A#{ProjectUploadFileFinder::ABSOLUTE_UPLOAD_DIR}/, LOST_AND_FOUND) + + move(path, new_path, 'move to lost and found', dry_run) + end + + def move(path, new_path, prefix, dry_run) + action = "#{prefix} #{path} -> #{new_path}" + + if dry_run + "Can #{action}" + else + begin + FileUtils.mkdir_p(File.dirname(new_path)) + FileUtils.mv(path, new_path) + + "Did #{action}" + rescue => e + "Error during #{action}: #{e.inspect}" + end + end + end + + # Yields absolute paths of project upload files that are not in the + # uploads table + def each_orphan_file + ProjectUploadFileFinder.new.each_file_batch do |file_paths| + logger.debug "Processing batch of #{file_paths.size} project upload file paths, starting with #{file_paths.first}" + + file_paths.each do |path| + pup = ProjectUploadPath.from_path(path) + + yield(path, pup.upload_path) if pup.orphan? + end + end + end + + class ProjectUploadPath + PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze + + attr_reader :full_path, :upload_path + + def initialize(full_path, upload_path) + @full_path = full_path + @upload_path = upload_path + end + + def self.from_path(path) + path_matched = path.match(PROJECT_FULL_PATH_REGEX) + return new(nil, nil) unless path_matched + + new(path_matched[1], path_matched[2]) + end + + def orphan? + return true if full_path.nil? || upload_path.nil? + + # It's possible to reduce to one query, but `where_full_path_in` is complex + !Upload.exists?(path: upload_path, model_id: project_id, model_type: 'Project', uploader: 'FileUploader') + end + + private + + def project_id + @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4fe5b4cc835..f39b3b6eb5b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -979,8 +979,8 @@ into similar problems in the future (e.g. when new tables are created). # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. - if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL - delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + if delay_interval < BackgroundMigrationWorker.minimum_interval + delay_interval = BackgroundMigrationWorker.minimum_interval end model_class.each_batch(of: batch_size) do |relation, index| diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb02c7ac8e1..73151e4a4c5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -39,19 +39,6 @@ module Gitlab ChecksumError = Class.new(StandardError) class << self - # Unlike `new`, `create` takes the repository path - def create(repo_path, bare: true, symlink_hooks_to: nil) - FileUtils.mkdir_p(repo_path, mode: 0770) - - # Equivalent to `git --git-path=#{repo_path} init [--bare]` - repo = Rugged::Repository.init_at(repo_path, bare) - repo.close - - create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present? - - true - end - def create_hooks(repo_path, global_hooks_path) local_hooks_path = File.join(repo_path, 'hooks') real_local_hooks_path = :not_found diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index fbe7d4ba1ae..41d58192818 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -166,7 +166,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.fast_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end diff --git a/lib/gitlab/graphs/commits.rb b/lib/gitlab/graphs/commits.rb index 3caf9036459..c4ffc19df09 100644 --- a/lib/gitlab/graphs/commits.rb +++ b/lib/gitlab/graphs/commits.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_per_day - @commit_per_day ||= @commits.size / (@duration + 1) + @commit_per_day ||= (@commits.size.to_f / (@duration + 1)).round(1) end def collect_data diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index 9251ed654cd..d11fcc6a3e3 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -30,7 +30,7 @@ module Gitlab end end - # Flag a project to me migrated + # Flag a project to be migrated # # @param [Object] project that will be migrated def migrate(project) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f163db936b..3adc44f8044 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -18,6 +18,21 @@ module Gitlab private + def download_or_copy_upload(uploader, upload_path) + if uploader.upload.local? + copy_files(uploader.path, upload_path) + else + download(uploader.url, upload_path) + end + end + + def download(url, upload_path) + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(url).open, file) + end + end + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4c411f4847e..7fd66b4e244 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,15 +10,18 @@ module Gitlab new(*args).import end - def initialize(archive_file:, shared:) + def initialize(project:, archive_file:, shared:) + @project = project @archive_file = archive_file @shared = shared end def import mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) - remove_symlinks! + remove_symlinks + copy_archive wait_for_archived_file do decompress_archive @@ -27,7 +30,8 @@ module Gitlab @shared.error(e) false ensure - remove_symlinks! + remove_import_file + remove_symlinks end private @@ -51,7 +55,15 @@ module Gitlab result end - def remove_symlinks! + def copy_archive + return if @archive_file + + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + + download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + end + + def remove_symlinks extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end @@ -59,6 +71,10 @@ module Gitlab true end + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index da3667faf7a..f69f98a78a3 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -107,6 +107,7 @@ excluded_attributes: - :storage_version - :remote_mirror_available_overridden - :description_html + - :repository_languages snippets: - :expired_at merge_request_diff: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 63cab07324a..4e179f63d8c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -35,7 +35,8 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + Gitlab::ImportExport::FileImporter.import(project: @project, + archive_file: @archive_file, shared: @shared) end @@ -91,7 +92,14 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + return unless Gitlab::ImportExport.object_storage? + + upload = @project.import_export_upload + + return unless upload&.import_file&.file + + upload.remove_import_file! + upload.save! end def overwrite_project diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 1110149712d..07875ebb56a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -91,10 +91,7 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(upload.file.url).open, file) - end + download_or_copy_upload(upload, upload_path) end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 5fc21878761..f7f5c5787f6 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -23,24 +23,28 @@ module Gitlab class << self def options - @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + Hash[import_table.map { |importer| [importer.title, importer.name] }] end def values - @values ||= ImportTable.map(&:name) + import_table.map(&:name) end def importer_names - @importer_names ||= ImportTable.select(&:importer).map(&:name) + import_table.select(&:importer).map(&:name) end def importer(name) - ImportTable.find { |import_source| import_source.name == name }.importer + import_table.find { |import_source| import_source.name == name }.importer end def title(name) options.key(name) end + + def import_table + ImportTable + end end end end diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 0f0588b8b23..530ccf88053 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,7 +1,7 @@ module Gitlab module Kubernetes module Helm - HELM_VERSION = '2.7.0'.freeze + HELM_VERSION = '2.7.2'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze end end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb new file mode 100644 index 00000000000..a41435fdb79 --- /dev/null +++ b/lib/gitlab/language_detection.rb @@ -0,0 +1,68 @@ +module Gitlab + class LanguageDetection + MAX_LANGUAGES = 5 + + def initialize(repository, repository_languages) + @repository = repository + @repository_languages = repository_languages + end + + def languages + detection.keys + end + + def language_color(name) + detection.dig(name, :color) + end + + # Newly detected languages, returned in a structure accepted by + # Gitlab::Database.bulk_insert + def insertions(programming_languages) + lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + + (languages - previous_language_names).map do |new_lang| + { + project_id: @repository.project.id, + share: detection[new_lang][:value], + programming_language_id: lang_to_id[new_lang] + } + end + end + + # updates analyses which records only require updating of their share + def updates + to_update = @repository_languages.select do |lang| + detection.key?(lang.name) && detection[lang.name][:value] != lang.share + end + + to_update.map do |lang| + { programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] } + end + end + + # Returns the ids of the programming languages that do not occur in the detection + # as current repository languages + def deletions + @repository_languages.map do |repo_lang| + next if detection.key?(repo_lang.name) + + repo_lang.programming_language_id + end.compact + end + + private + + def previous_language_names + @previous_language_names ||= @repository_languages.map(&:name) + end + + def detection + @detection ||= + @repository + .languages + .first(MAX_LANGUAGES) + .map { |l| [l[:label], l] } + .to_h + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index e1a958c508a..0f26fcfe8cb 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -99,5 +99,9 @@ module Gitlab ) }mx end + + def jira_transition_id_regex + @jira_transition_id_regex ||= /\d+/ + end end end diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb new file mode 100644 index 00000000000..f24a01e6cf5 --- /dev/null +++ b/lib/gitlab/template_helper.rb @@ -0,0 +1,28 @@ +module Gitlab + module TemplateHelper + include Gitlab::Utils::StrongMemoize + + def prepare_template_environment(file) + return unless file + + if Gitlab::ImportExport.object_storage? + params[:import_export_upload] = ImportExportUpload.new(import_file: file) + else + FileUtils.mkdir_p(File.dirname(import_upload_path)) + FileUtils.copy_entry(file.path, import_upload_path) + + params[:import_source] = import_upload_path + end + end + + def import_upload_path + strong_memoize(:import_upload_path) do + Gitlab::ImportExport.import_upload_path(filename: tmp_filename) + end + end + + def tmp_filename + SecureRandom.hex + end + end +end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 5e07b12ee1c..a2feb074b1d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -7,9 +7,8 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) + namespaces = Namespace.pluck(:path) namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored Gitlab.config.repositories.storages.each do |name, repository_storage| git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } @@ -31,7 +30,7 @@ namespace :gitlab do end all_dirs.each do |dir_path| - if remove_flag + if remove? if FileUtils.rm_rf dir_path puts "Removed...#{dir_path}".color(:red) else @@ -43,7 +42,7 @@ namespace :gitlab do end end - unless remove_flag + unless remove? puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) end end @@ -104,5 +103,37 @@ namespace :gitlab do puts "To block these users run this command with BLOCK=true".color(:yellow) end end + + desc "GitLab | Cleanup | Clean orphaned project uploads" + task project_uploads: :gitlab_environment do + warn_user_is_not_gitlab + + cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger) + cleaner.run!(dry_run: dry_run?) + + if dry_run? + logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow) + end + end + + def remove? + ENV['REMOVE'] == 'true' + end + + def dry_run? + ENV['DRY_RUN'] != 'false' + end + + def logger + return @logger if defined?(@logger) + + @logger = if Rails.env.development? || Rails.env.production? + Logger.new(STDOUT).tap do |stdout_logger| + stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger)) + end + else + Rails.logger + end + end end end |