diff options
29 files changed, 332 insertions, 106 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1d78ef2d48..be18520b876 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -321,6 +321,7 @@ setup-test-env: expire_in: 7d paths: - tmp/tests + - config/secrets.yml rspec-pg 0 27: *rspec-metadata-pg rspec-pg 1 27: *rspec-metadata-pg @@ -325,7 +325,7 @@ group :development, :test do gem 'spinach-rerun-reporter', '~> 0.0.2' gem 'rspec_profiling', '~> 0.0.5' gem 'rspec-set', '~> 0.1.3' - gem 'rspec-parameterized' + gem 'rspec-parameterized', require: false # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826) gem 'minitest', '~> 5.7.0' diff --git a/Gemfile.lock b/Gemfile.lock index 5532888d179..1a3c8f42469 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -304,7 +304,7 @@ GEM mime-types (>= 1.16) posix-spawn (~> 0.3) gitlab-markup (1.6.3) - gitlab-styles (2.3.1) + gitlab-styles (2.3.2) rubocop (~> 0.51) rubocop-gitlab-security (~> 0.1.0) rubocop-rspec (~> 1.19) diff --git a/bin/upgrade.rb b/bin/upgrade.rb deleted file mode 100755 index a5caecf8526..00000000000 --- a/bin/upgrade.rb +++ /dev/null @@ -1,3 +0,0 @@ -require_relative "../lib/gitlab/upgrader" - -Gitlab::Upgrader.new.execute diff --git a/config/application.rb b/config/application.rb index 2067428ff62..751307de975 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,6 +6,7 @@ Bundler.require(:default, Rails.env) module Gitlab class Application < Rails::Application + require_dependency Rails.root.join('lib/gitlab/redis/wrapper') require_dependency Rails.root.join('lib/gitlab/redis/cache') require_dependency Rails.root.join('lib/gitlab/redis/queues') require_dependency Rails.root.join('lib/gitlab/redis/shared_state') diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index 4bc5cda8cb5..b9832a724c4 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,7 +5,17 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil, vars = {}) + Result = Struct.new(:cmd, :stdout, :stderr, :status, :duration) + + # Returns [stdout + stderr, status] + def popen(cmd, path = nil, vars = {}, &block) + result = popen_with_detail(cmd, path, vars, &block) + + [result.stdout << result.stderr, result.status&.exitstatus] + end + + # Returns Result + def popen_with_detail(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end @@ -18,18 +28,21 @@ module Gitlab FileUtils.mkdir_p(path) end - cmd_output = "" - cmd_status = 0 + cmd_stdout = '' + cmd_stderr = '' + cmd_status = nil + start = Time.now + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| yield(stdin) if block_given? stdin.close - cmd_output << stdout.read - cmd_output << stderr.read - cmd_status = wait_thr.value.exitstatus + cmd_stdout = stdout.read + cmd_stderr = stderr.read + cmd_status = wait_thr.value end - [cmd_output, cmd_status] + Result.new(cmd, cmd_stdout, cmd_stderr, cmd_status, Time.now - start) end end end diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb new file mode 100644 index 00000000000..f44035a48bb --- /dev/null +++ b/lib/gitlab/popen/runner.rb @@ -0,0 +1,46 @@ +module Gitlab + module Popen + class Runner + attr_reader :results + + def initialize + @results = [] + end + + def run(commands, &block) + commands.each do |cmd| + # yield doesn't support blocks, so we need to use a block variable + block.call(cmd) do # rubocop:disable Performance/RedundantBlockCall + cmd_result = Gitlab::Popen.popen_with_detail(cmd) + + results << cmd_result + + cmd_result + end + end + end + + def all_success_and_clean? + all_success? && all_stderr_empty? + end + + def all_success? + results.all? { |result| result.status.success? } + end + + def all_stderr_empty? + results.all? { |result| result.stderr.empty? } + end + + def failed_results + results.reject { |result| result.status.success? } + end + + def warned_results + results.select do |result| + result.status.success? && !result.stderr.empty? + end + end + end + end +end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 9bf019b72e6..a991933e910 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,5 +1,5 @@ # please require all dependencies below: -require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) +require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? module Gitlab module Redis diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index c1182af1014..34bee6fecbe 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,6 +1,7 @@ require 'rainbow/ext/string' require 'gitlab/utils/strong_memoize' +# rubocop:disable Rails/Output module Gitlab TaskFailedError = Class.new(StandardError) TaskAbortedByUserError = Class.new(StandardError) @@ -96,11 +97,9 @@ module Gitlab end def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" end def gitlab_user diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 3b64cb32afa..d545f2f95f1 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -1,6 +1,3 @@ -require_relative "popen" -require_relative "version_info" - module Gitlab class Upgrader def execute diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index c42ae4fe4c4..914ed794601 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -1,5 +1,3 @@ -require 'tasks/gitlab/task_helpers' - module SystemCheck module Helpers include ::Gitlab::TaskHelpers diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 7ad2b2e4d39..b1e012e70c5 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/` + output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` if output.include? "Similar code found" puts output diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2383bcf954b..24e37f6c6cc 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -4,7 +4,7 @@ namespace :gitlab do namespace :backup do # Create backup of GitLab system desc "GitLab | Create a backup of the GitLab system" - task create: :environment do + task create: :gitlab_environment do warn_user_is_not_gitlab configure_cron_mode @@ -25,7 +25,7 @@ namespace :gitlab do # Restore backup of GitLab system desc 'GitLab | Restore a previously created backup' - task restore: :environment do + task restore: :gitlab_environment do warn_user_is_not_gitlab configure_cron_mode @@ -73,7 +73,7 @@ namespace :gitlab do end namespace :repo do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") @@ -84,7 +84,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring repositories ...".color(:blue) Backup::Repository.new.restore $progress.puts "done".color(:green) @@ -92,7 +92,7 @@ namespace :gitlab do end namespace :db do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") @@ -103,7 +103,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring database ... ".color(:blue) Backup::Database.new.restore $progress.puts "done".color(:green) @@ -111,7 +111,7 @@ namespace :gitlab do end namespace :builds do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") @@ -122,7 +122,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring builds ... ".color(:blue) Backup::Builds.new.restore $progress.puts "done".color(:green) @@ -130,7 +130,7 @@ namespace :gitlab do end namespace :uploads do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") @@ -141,7 +141,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring uploads ... ".color(:blue) Backup::Uploads.new.restore $progress.puts "done".color(:green) @@ -149,7 +149,7 @@ namespace :gitlab do end namespace :artifacts do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") @@ -160,7 +160,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring artifacts ... ".color(:blue) Backup::Artifacts.new.restore $progress.puts "done".color(:green) @@ -168,7 +168,7 @@ namespace :gitlab do end namespace :pages do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping pages ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("pages") @@ -179,7 +179,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring pages ... ".color(:blue) Backup::Pages.new.restore $progress.puts "done".color(:green) @@ -187,7 +187,7 @@ namespace :gitlab do end namespace :lfs do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") @@ -198,7 +198,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring lfs objects ... ".color(:blue) Backup::Lfs.new.restore $progress.puts "done".color(:green) @@ -206,7 +206,7 @@ namespace :gitlab do end namespace :registry do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping container registry images ... ".color(:blue) if Gitlab.config.registry.enabled @@ -221,7 +221,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring container registry images ... ".color(:blue) if Gitlab.config.registry.enabled diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a584eb97cf5..e05a3aad824 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,7 +1,3 @@ -# Temporary hack, until we migrate all checks to SystemCheck format -require 'system_check' -require 'system_check/helpers' - namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check @@ -12,7 +8,7 @@ namespace :gitlab do namespace :app do desc 'GitLab | Check the configuration of the GitLab Rails app' - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab checks = [ @@ -43,7 +39,7 @@ namespace :gitlab do namespace :gitlab_shell do desc "GitLab | Check the configuration of GitLab Shell" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab start_checking "GitLab Shell" @@ -251,7 +247,7 @@ namespace :gitlab do namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab start_checking "Sidekiq" @@ -310,7 +306,7 @@ namespace :gitlab do namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab if Gitlab.config.incoming_email.enabled @@ -333,7 +329,7 @@ namespace :gitlab do end namespace :ldap do - task :check, [:limit] => :environment do |_, args| + task :check, [:limit] => :gitlab_environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. args.with_defaults(limit: 100) @@ -389,7 +385,7 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" - task check: :environment do + task check: :gitlab_environment do puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) Rake::Task["gitlab:git:fsck"].execute end @@ -397,7 +393,7 @@ namespace :gitlab do namespace :orphans do desc 'Gitlab | Check for orphaned namespaces and repositories' - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab checks = [ SystemCheck::Orphans::NamespaceCheck, @@ -408,7 +404,7 @@ namespace :gitlab do end desc 'GitLab | Check for orphaned namespaces in the repositories path' - task check_namespaces: :environment do + task check_namespaces: :gitlab_environment do warn_user_is_not_gitlab checks = [SystemCheck::Orphans::NamespaceCheck] @@ -416,7 +412,7 @@ namespace :gitlab do end desc 'GitLab | Check for orphaned repositories in the repositories path' - task check_repositories: :environment do + task check_repositories: :gitlab_environment do warn_user_is_not_gitlab checks = [SystemCheck::Orphans::RepositoryCheck] @@ -426,7 +422,7 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" - task :check_repos, [:username] => :environment do |t, args| + task :check_repos, [:username] => :gitlab_environment do |t, args| username = args[:username] || prompt("Check repository integrity for username? ".color(:blue)) user = User.find_by(username: username) if user diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab601b0d66b..5a53eac0897 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,7 +5,7 @@ namespace :gitlab do HASHED_REPOSITORY_NAME = '@hashed'.freeze desc "GitLab | Cleanup | Clean namespaces" - task dirs: :environment do + task dirs: :gitlab_environment do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] @@ -49,7 +49,7 @@ namespace :gitlab do end desc "GitLab | Cleanup | Clean repositories" - task repos: :environment do + task repos: :gitlab_environment do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" @@ -78,7 +78,7 @@ namespace :gitlab do end desc "GitLab | Cleanup | Block users that have been removed in LDAP" - task block_removed_ldap_users: :environment do + task block_removed_ldap_users: :gitlab_environment do warn_user_is_not_gitlab block_flag = ENV['BLOCK'] @@ -109,7 +109,7 @@ namespace :gitlab do # released. So likely this should only be run once on gitlab.com # Faulty refs are moved so they are kept around, else some features break. desc 'GitLab | Cleanup | Remove faulty deployment refs' - task move_faulty_deployment_refs: :environment do + task move_faulty_deployment_refs: :gitlab_environment do projects = Project.where(id: Deployment.select(:project_id).distinct) projects.find_each do |project| diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 3f5dd2ae3b3..cb4f7e5c8a8 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :git do desc "GitLab | Git | Repack" - task repack: :environment do + task repack: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".color(:green) @@ -11,7 +11,7 @@ namespace :gitlab do end desc "GitLab | Git | Run garbage collection on all repos" - task gc: :environment do + task gc: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".color(:green) @@ -21,7 +21,7 @@ namespace :gitlab do end desc "GitLab | Git | Prune all repos" - task prune: :environment do + task prune: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") if failures.empty? puts "Done".color(:green) @@ -31,7 +31,7 @@ namespace :gitlab do end desc 'GitLab | Git | Check all repos integrity' - task fsck: :environment do + task fsck: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| check_config_lock(repo) check_ref_locks(repo) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index aece8893974..107ff1d8aeb 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir, :repo] => :environment do |t, args| + task :install, [:dir, :repo] => :gitlab_environment do |t, args| require 'toml' warn_user_is_not_gitlab diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index b0a24790c4a..14d1125a03d 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -1,8 +1,6 @@ -require 'tasks/gitlab/task_helpers' - # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] -namespace :gitlab do +task gitlab_environment: :environment do extend SystemCheck::Helpers end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e9fb6a008b0..45e9a1a1c72 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :env do desc "GitLab | Show information about GitLab and its environment" - task info: :environment do + task info: :gitlab_environment do # check if there is an RVM environment rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s) # check Ruby version diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 05fcb8e3da5..1d903c81358 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -1,6 +1,6 @@ namespace :gitlab do desc "GitLab | Setup production application" - task setup: :environment do + task setup: :gitlab_environment do setup_db end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 12ae4199b69..844664b12d4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:repo] => :environment do |t, args| + task :install, [:repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required @@ -58,12 +58,12 @@ namespace :gitlab do end desc "GitLab | Setup gitlab-shell" - task setup: :environment do + task setup: :gitlab_environment do setup end desc "GitLab | Build missing projects" - task build_missing_projects: :environment do + task build_missing_projects: :gitlab_environment do Project.find_each(batch_size: 1000) do |project| path_to_repo = project.repository.path_to_repo if File.exist?(path_to_repo) @@ -80,7 +80,7 @@ namespace :gitlab do end desc 'Create or repair repository hooks symlink' - task create_hooks: :environment do + task create_hooks: :gitlab_environment do warn_user_is_not_gitlab puts 'Creating/Repairing hooks symlinks for all repositories' diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index 308ffb0e284..b917a293095 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir, :repo] => :environment do |t, args| + task :install, [:dir, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index ad2d034b0b4..5c0cc4990fc 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -2,5 +2,14 @@ unless Rails.env.production? require 'haml_lint/rake_task' require 'haml_lint/inline_javascript' + # Workaround for warnings from parser/current + # TODO: Remove this after we update parser gem + task :haml_lint do + require 'parser' + def Parser.warn(*args) + puts(*args) # static-analysis ignores stdout if status is 0 + end + end + HamlLint::RakeTask.new end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index c996537cfbe..31cbd651edb 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,16 +1,14 @@ -require Rails.root.join('lib/gitlab/database') -require Rails.root.join('lib/gitlab/database/migration_helpers') -require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') -require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') -require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') -require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') -require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') -require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') -require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') -require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') - desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do + require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') + require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') + require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') + require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') + require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') + require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') + require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') + require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up diff --git a/scripts/static-analysis b/scripts/static-analysis index 96d08287ded..b3895292ab4 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -1,6 +1,29 @@ #!/usr/bin/env ruby -require ::File.expand_path('../lib/gitlab/popen', __dir__) +# We don't have auto-loading here +require_relative '../lib/gitlab/popen' +require_relative '../lib/gitlab/popen/runner' + +def emit_warnings(static_analysis) + static_analysis.warned_results.each do |result| + puts + puts "**** #{result.cmd.join(' ')} had the following warnings:" + puts + puts result.stderr + puts + end +end + +def emit_errors(static_analysis) + static_analysis.failed_results.each do |result| + puts + puts "**** #{result.cmd.join(' ')} failed with the following error:" + puts + puts result.stdout + puts result.stderr + puts + end +end tasks = [ %w[bundle exec rake config_lint], @@ -17,18 +40,16 @@ tasks = [ %w[scripts/lint-rugged] ] -failed_tasks = tasks.reduce({}) do |failures, task| - start = Time.now - puts - puts "$ #{task.join(' ')}" +static_analysis = Gitlab::Popen::Runner.new - output, status = Gitlab::Popen.popen(task) - puts "==> Finished in #{Time.now - start} seconds" +static_analysis.run(tasks) do |cmd, &run| puts + puts "$ #{cmd.join(' ')}" - failures[task.join(' ')] = output unless status.zero? + result = run.call - failures + puts "==> Finished in #{result.duration} seconds" + puts end puts @@ -36,17 +57,20 @@ puts '===================================================' puts puts -if failed_tasks.empty? +if static_analysis.all_success_and_clean? puts 'All static analyses passed successfully.' +elsif static_analysis.all_success? + puts 'All static analyses passed successfully, but we have warnings:' + puts + + emit_warnings(static_analysis) + + exit 2 else puts 'Some static analyses failed:' - failed_tasks.each do |failed_task, output| - puts - puts "**** #{failed_task} failed with the following error:" - puts - puts output - end + emit_warnings(static_analysis) + emit_errors(static_analysis) exit 1 end diff --git a/spec/lib/gitlab/popen/runner_spec.rb b/spec/lib/gitlab/popen/runner_spec.rb new file mode 100644 index 00000000000..2e2cb4ca28f --- /dev/null +++ b/spec/lib/gitlab/popen/runner_spec.rb @@ -0,0 +1,139 @@ +require 'spec_helper' + +describe Gitlab::Popen::Runner do + subject { described_class.new } + + describe '#run' do + it 'runs the command and returns the result' do + run_command + + expect(Gitlab::Popen).to have_received(:popen_with_detail) + end + end + + describe '#all_success_and_clean?' do + it 'returns true when exit status is 0 and stderr is empty' do + run_command + + expect(subject).to be_all_success_and_clean + end + + it 'returns false when exit status is not 0' do + run_command(exitstatus: 1) + + expect(subject).not_to be_all_success_and_clean + end + + it 'returns false when exit stderr has something' do + run_command(stderr: 'stderr') + + expect(subject).not_to be_all_success_and_clean + end + end + + describe '#all_success?' do + it 'returns true when exit status is 0' do + run_command + + expect(subject).to be_all_success + end + + it 'returns false when exit status is not 0' do + run_command(exitstatus: 1) + + expect(subject).not_to be_all_success + end + + it 'returns true' do + run_command(stderr: 'stderr') + + expect(subject).to be_all_success + end + end + + describe '#all_stderr_empty?' do + it 'returns true when stderr is empty' do + run_command + + expect(subject).to be_all_stderr_empty + end + + it 'returns true when exit status is not 0' do + run_command(exitstatus: 1) + + expect(subject).to be_all_stderr_empty + end + + it 'returns false when exit stderr has something' do + run_command(stderr: 'stderr') + + expect(subject).not_to be_all_stderr_empty + end + end + + describe '#failed_results' do + it 'returns [] when everything is passed' do + run_command + + expect(subject.failed_results).to be_empty + end + + it 'returns the result when exit status is not 0' do + result = run_command(exitstatus: 1) + + expect(subject.failed_results).to contain_exactly(result) + end + + it 'returns [] when exit stderr has something' do + run_command(stderr: 'stderr') + + expect(subject.failed_results).to be_empty + end + end + + describe '#warned_results' do + it 'returns [] when everything is passed' do + run_command + + expect(subject.warned_results).to be_empty + end + + it 'returns [] when exit status is not 0' do + run_command(exitstatus: 1) + + expect(subject.warned_results).to be_empty + end + + it 'returns the result when exit stderr has something' do + result = run_command(stderr: 'stderr') + + expect(subject.warned_results).to contain_exactly(result) + end + end + + def run_command( + command: 'command', + stdout: 'stdout', + stderr: '', + exitstatus: 0, + status: double(exitstatus: exitstatus, success?: exitstatus.zero?), + duration: 0.1) + + result = + Gitlab::Popen::Result.new(command, stdout, stderr, status, duration) + + allow(Gitlab::Popen) + .to receive(:popen_with_detail) + .and_return(result) + + subject.run([command]) do |cmd, &run| + expect(cmd).to eq(command) + + cmd_result = run.call + + expect(cmd_result).to eq(result) + end + + subject.results.first + end +end diff --git a/spec/lib/gitlab/popen_spec.rb b/spec/lib/gitlab/popen_spec.rb index b145ca36f26..1dbead16d5b 100644 --- a/spec/lib/gitlab/popen_spec.rb +++ b/spec/lib/gitlab/popen_spec.rb @@ -1,11 +1,23 @@ require 'spec_helper' -describe 'Gitlab::Popen' do +describe Gitlab::Popen do let(:path) { Rails.root.join('tmp').to_s } before do @klass = Class.new(Object) - @klass.send(:include, Gitlab::Popen) + @klass.send(:include, described_class) + end + + describe '.popen_with_detail' do + subject { @klass.new.popen_with_detail(cmd) } + + let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1);$stderr.puts(2);exit(3)] } + + it { expect(subject.cmd).to eq(cmd) } + it { expect(subject.stdout).to eq("1\n") } + it { expect(subject.stderr).to eq("2\n") } + it { expect(subject.status.exitstatus).to eq(3) } + it { expect(subject.duration).to be_kind_of(Numeric) } end context 'zero status' do diff --git a/spec/support/javascript_fixtures_helpers.rb b/spec/support/javascript_fixtures_helpers.rb index 923c8080e6c..2197bc9d853 100644 --- a/spec/support/javascript_fixtures_helpers.rb +++ b/spec/support/javascript_fixtures_helpers.rb @@ -1,6 +1,5 @@ require 'action_dispatch/testing/test_request' require 'fileutils' -require 'gitlab/popen' module JavaScriptFixturesHelpers include Gitlab::Popen diff --git a/spec/tasks/gitlab/task_helpers_spec.rb b/spec/tasks/gitlab/task_helpers_spec.rb index fae5ec35c47..e9322ec4931 100644 --- a/spec/tasks/gitlab/task_helpers_spec.rb +++ b/spec/tasks/gitlab/task_helpers_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'tasks/gitlab/task_helpers' class TestHelpersTest include Gitlab::TaskHelpers |