diff options
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 7 | ||||
| -rw-r--r-- | doc/ci/docker/using_docker_images.md | 4 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/gitaly_client/storage_settings.rb | 2 | ||||
| -rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 2 | ||||
| -rw-r--r-- | lib/tasks/gitlab/helpers.rake | 4 | ||||
| -rwxr-xr-x | scripts/setup-test-env | 66 | ||||
| -rw-r--r-- | spec/spec_helper.rb | 4 | ||||
| -rw-r--r-- | spec/support/helpers/test_env.rb | 78 |
9 files changed, 116 insertions, 53 deletions
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index c89dc0ab13f..c86981b1095 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -149,13 +149,16 @@ setup-test-env: - .rails-job-base - .setup-test-env-cache - .rails:rules:code-backstage-qa - - .use-pg12 stage: prepare variables: GITLAB_TEST_EAGER_LOAD: "0" + SETUP_DB: "false" script: - - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" + - run_timed_command "scripts/setup-test-env" + - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries" - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here + - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K" + artifacts: expire_in: 7d paths: diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 173701ef358..29f4053f720 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -128,6 +128,10 @@ For example, the following two definitions are equal: - name: redis:latest ``` +## Where scripts are executed + +When a CI job runs in a Docker container, the `before_script`, `script`, and `after_script` commands run in the `/builds/<project-path>/` directory. Your image may have a different default `WORKDIR` defined. To move to your `WORKDIR`, save the `WORKDIR` as an environment variable so you can reference it in the container during the job's runtime. + ### Available settings for `image` > Introduced in GitLab and GitLab Runner 9.4. diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f4a89edecd1..2c26da037da 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -120,7 +120,7 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix tls)) + unless %w(tcp unix tls).include?(URI(address).scheme) raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index f66dc3010ea..4cc0269673f 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -52,7 +52,7 @@ module Gitlab @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] storage['path'] = Deprecated - @hash = storage.with_indifferent_access + @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) end def gitaly_address diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index df75b3cf716..cafa06e2a67 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -3,7 +3,7 @@ namespace :gitlab do namespace :gitaly do desc 'GitLab | Gitaly | Install or upgrade gitaly' - task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| + task :install, [:dir, :storage_path, :repo] => :with_gitlab_helpers do |t, args| warn_user_is_not_gitlab unless args.dir.present? && args.storage_path.present? diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index b61b1833c5a..220a333fbd6 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -6,3 +6,7 @@ StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] task gitlab_environment: :environment do extend SystemCheck::Helpers end + +task :with_gitlab_helpers do + extend SystemCheck::Helpers +end diff --git a/scripts/setup-test-env b/scripts/setup-test-env new file mode 100755 index 00000000000..725f1706380 --- /dev/null +++ b/scripts/setup-test-env @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +require 'bundler/setup' + +require 'request_store' +require 'rake' +require 'active_support/dependencies' +require 'active_support/dependencies/autoload' +require 'active_support/core_ext/numeric' +require 'active_support/string_inquirer' + +module Rails + extend self + + def root + Pathname.new(File.expand_path('..', __dir__)) + end + + def env + @_env ||= ActiveSupport::StringInquirer.new(ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "test") + end +end + +ActiveSupport::Dependencies.autoload_paths << 'lib' + +load File.expand_path('../lib/tasks/gitlab/helpers.rake', __dir__) +load File.expand_path('../lib/tasks/gitlab/gitaly.rake', __dir__) + +# Required for config/0_inject_enterprise_edition_module.rb, lib/gitlab/access.rb +require_dependency File.expand_path('../lib/gitlab', __dir__) + +require_dependency File.expand_path('../config/initializers/0_inject_enterprise_edition_module', __dir__) + +# Require for lib/gitlab/gitaly_client/storage_settings.rb and config/initializers/1_settings.rb +require 'active_support/hash_with_indifferent_access' + +# Required for lib/gitlab/visibility_level.rb and lib/gitlab/safe_request_store.rb +require 'active_support/concern' +require 'active_support/core_ext/module/delegation' + +# Required for lib/system_check/helpers.rb +require_dependency File.expand_path('../lib/gitlab/task_helpers', __dir__) + +# Required for lib/tasks/gitlab/helpers.rake +require_dependency File.expand_path('../lib/system_check/helpers', __dir__) + +# Required for config/initializers/1_settings.rb +require 'omniauth' +require 'omniauth-github' +require 'etc' +require_dependency File.expand_path('../lib/gitlab/access', __dir__) + +require_dependency File.expand_path('../config/initializers/1_settings', __dir__) + +Gitlab.ee do + load File.expand_path('../ee/lib/tasks/gitlab/indexer.rake', __dir__) + + require_dependency File.expand_path('../ee/lib/gitlab/elastic/indexer', __dir__) + require_dependency File.expand_path('../lib/gitlab/utils/override', __dir__) +end + +require_dependency File.expand_path('../spec/support/helpers/test_env', __dir__) + +TestEnv.init diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bd9ba53c04c..1c852b85ff5 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -230,6 +230,10 @@ RSpec.configure do |config| Gitlab::Database.set_open_transactions_baseline end + config.append_before do + Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group] + end + config.append_after do Gitlab::Database.reset_open_transactions_baseline end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 40a3dbfbf25..29aefbeb99c 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true +require 'parallel' + module TestEnv - extend ActiveSupport::Concern extend self ComponentFailedToInstallError = Class.new(StandardError) @@ -94,50 +95,40 @@ module TestEnv TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze REPOS_STORAGE = 'default' SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage') + SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze + + # Can be overriden + def setup_methods + SETUP_METHODS + end # Test environment # # See gitlab.yml.example test section for paths # - def init(opts = {}) + def init unless Rails.env.test? puts "\nTestEnv.init can only be run if `RAILS_ENV` is set to 'test' not '#{Rails.env}'!\n" exit 1 end + start = Time.now # Disable mailer for spinach tests - disable_mailer if opts[:mailer] == false - clean_test_path - setup_gitlab_shell - - setup_gitaly - - # Feature specs are run through Workhorse - setup_workhorse - - # Create repository for FactoryBot.create(:project) - setup_factory_repo - - # Create repository for FactoryBot.create(:forked_project_with_submodules) - setup_forked_repo - end - - included do |config| - config.append_before do - set_current_example_group + # Install components in parallel as most of the setup is I/O. + Parallel.each(setup_methods) do |method| + public_send(method) end - end - def disable_mailer - allow_any_instance_of(NotificationService).to receive(:mailer) - .and_return(double.as_null_object) + post_init + + puts "\nTest environment set up in #{Time.now - start} seconds" end - def enable_mailer - allow_any_instance_of(NotificationService).to receive(:mailer) - .and_call_original + # Can be overriden + def post_init + start_gitaly(gitaly_dir) end # Clean /tmp/tests @@ -164,12 +155,11 @@ module TestEnv end def setup_gitaly - install_gitaly_args = [gitaly_dir, repos_path, gitaly_url].compact.join(',') - component_timed_setup('Gitaly', install_dir: gitaly_dir, version: Gitlab::GitalyClient.expected_server_version, - task: "gitlab:gitaly:install[#{install_gitaly_args}]") do + task: "gitlab:gitaly:install", + task_args: [gitaly_dir, repos_path, gitaly_url].compact) do Gitlab::SetupHelper::Gitaly.create_configuration( gitaly_dir, { 'default' => repos_path }, @@ -190,8 +180,6 @@ module TestEnv ) Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true) end - - start_gitaly(gitaly_dir) end def gitaly_socket_path @@ -273,19 +261,18 @@ module TestEnv raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds" end + # Feature specs are run through Workhorse def setup_workhorse start = Time.now return if skip_compile_workhorse? - puts "\n==> Setting up GitLab Workhorse..." - FileUtils.rm_rf(workhorse_dir) Gitlab::SetupHelper::Workhorse.compile_into(workhorse_dir) Gitlab::SetupHelper::Workhorse.create_configuration(workhorse_dir, nil) File.write(workhorse_tree_file, workhorse_tree) if workhorse_source_clean? - puts " GitLab Workhorse set up in #{Time.now - start} seconds...\n" + puts "==> GitLab Workhorse set up in #{Time.now - start} seconds...\n" end def skip_compile_workhorse? @@ -349,10 +336,12 @@ module TestEnv ENV.fetch('GITLAB_WORKHORSE_URL', nil) end + # Create repository for FactoryBot.create(:project) def setup_factory_repo setup_repo(factory_repo_path, factory_repo_path_bare, factory_repo_name, BRANCH_SHA) end + # Create repository for FactoryBot.create(:forked_project_with_submodules) # This repo has a submodule commit that is not present in the main test # repository. def setup_forked_repo @@ -363,20 +352,18 @@ module TestEnv clone_url = "https://gitlab.com/gitlab-org/#{repo_name}.git" unless File.directory?(repo_path) - puts "\n==> Setting up #{repo_name} repository in #{repo_path}..." start = Time.now system(*%W(#{Gitlab.config.git.bin_path} clone --quiet -- #{clone_url} #{repo_path})) - puts " #{repo_path} set up in #{Time.now - start} seconds...\n" + puts "==> #{repo_path} set up in #{Time.now - start} seconds...\n" end set_repo_refs(repo_path, refs) unless File.directory?(repo_path_bare) - puts "\n==> Setting up #{repo_name} bare repository in #{repo_path_bare}..." start = Time.now # We must copy bare repositories because we will push to them. system(git_env, *%W(#{Gitlab.config.git.bin_path} clone --quiet --bare -- #{repo_path} #{repo_path_bare})) - puts " #{repo_path_bare} set up in #{Time.now - start} seconds...\n" + puts "==> #{repo_path_bare} set up in #{Time.now - start} seconds...\n" end end @@ -468,10 +455,6 @@ module TestEnv private - def set_current_example_group - Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group] - end - # These are directories that should be preserved at cleanup time def test_dirs @test_dirs ||= %w[ @@ -526,7 +509,7 @@ module TestEnv end end - def component_timed_setup(component, install_dir:, version:, task:) + def component_timed_setup(component, install_dir:, version:, task:, task_args: []) start = Time.now ensure_component_dir_name_is_correct!(component, install_dir) @@ -535,17 +518,16 @@ module TestEnv return if File.exist?(install_dir) && ci? if component_needs_update?(install_dir, version) - puts "\n==> Setting up #{component}..." # Cleanup the component entirely to ensure we start fresh FileUtils.rm_rf(install_dir) - unless system('rake', task) + unless Rake::Task[task].invoke(*task_args) raise ComponentFailedToInstallError end yield if block_given? - puts " #{component} set up in #{Time.now - start} seconds...\n" + puts "==> #{component} set up in #{Time.now - start} seconds...\n" end rescue ComponentFailedToInstallError puts "\n#{component} failed to install, cleaning up #{install_dir}!\n" |