diff options
Diffstat (limited to 'scripts')
24 files changed, 532 insertions, 97 deletions
diff --git a/scripts/build_assets_image b/scripts/build_assets_image index b799947f73d..4b192ed5e22 100755 --- a/scripts/build_assets_image +++ b/scripts/build_assets_image @@ -33,8 +33,8 @@ if ([ "${CI_PROJECT_NAME}" = "gitlab" ] && [ "${FOSS_ONLY}" != "1" ]) || ([ "${C ASSETS_IMAGE_NAME="gitlab-assets-ee" fi -# Generate this image for https://jihulab.com/gitlab-cn/gitlab and https://gitlab.com/gitlab-jh/jh-team/gitlab -if ([ "${CI_PROJECT_NAMESPACE}" = "gitlab-cn" ] || [ "${CI_PROJECT_NAMESPACE}" = "gitlab-jh" ]); then +# Generate this image for https://jihulab.com/gitlab-cn/gitlab +if [ "${CI_PROJECT_NAMESPACE}" = "gitlab-cn" ]; then ASSETS_IMAGE_NAME="gitlab-assets-jh" fi diff --git a/scripts/database/migration_collision_checker.rb b/scripts/database/migration_collision_checker.rb new file mode 100644 index 00000000000..0c9daee9f4c --- /dev/null +++ b/scripts/database/migration_collision_checker.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'pathname' +require 'open3' + +# Checks for class name collisions between Database migrations and Elasticsearch migrations +class MigrationCollisionChecker + MIGRATION_FOLDERS = %w[db/migrate/*.rb db/post_migrate/*.rb ee/elastic/migrate/*.rb].freeze + + CLASS_MATCHER = /^\s*class\s+:*([A-Z][A-Za-z0-9_]+\S+)/ + + ERROR_CODE = 1 + + # To be removed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129012 + SKIP_MIGRATIONS = %w[AddInternalToNotes BackfillInternalOnNotes].freeze + + Result = Struct.new(:error_code, :error_message) + + def initialize + @collisions = Hash.new { |h, k| h[k] = [] } + end + + def check + check_for_collisions + + return if collisions.empty? + + Result.new(ERROR_CODE, "\e[31mError: Naming collisions were found between migrations\n\n#{message}\e[0m") + end + + private + + attr_reader :collisions + + def check_for_collisions + MIGRATION_FOLDERS.each do |migration_folder| + Dir.glob(base_path.join(migration_folder)).each do |migration_path| + klass_name = CLASS_MATCHER.match(File.read(migration_path))[1] + + next if SKIP_MIGRATIONS.include?(klass_name) + + collisions[klass_name] << migration_path + end + end + + collisions.select! { |_, v| v.size > 1 } + end + + def message + collisions.map { |klass_name, paths| "#{klass_name}: #{paths.join(', ')}\n" }.join('') + end + + def base_path + Pathname.new(File.expand_path('../../', __dir__)) + end +end diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key index fbffebb6086..66781343411 100755 --- a/scripts/decomposition/generate-loose-foreign-key +++ b/scripts/decomposition/generate-loose-foreign-key @@ -164,18 +164,20 @@ def generate_migration(definition) class Remove#{definition.to_table.camelcase}#{definition.from_table.camelcase}#{definition.column.camelcase}Fk < Gitlab::Database::Migration[2.1] disable_ddl_transaction! + FOREIGN_KEY_NAME = "#{definition.name}" + def up - return unless foreign_key_exists?(:#{definition.from_table}, :#{definition.to_table}, name: "#{definition.name}") + return unless foreign_key_exists?(:#{definition.from_table}, :#{definition.to_table}, name: FOREIGN_KEY_NAME) with_lock_retries do remove_foreign_key_if_exists(:#{definition.from_table}, :#{definition.to_table}, - name: "#{definition.name}", reverse_lock_order: true) + name: FOREIGN_KEY_NAME, reverse_lock_order: true) end end def down add_concurrent_foreign_key(:#{definition.from_table}, :#{definition.to_table}, - name: "#{definition.name}", column: :#{definition.column}, + name: FOREIGN_KEY_NAME, column: :#{definition.column}, target_column: :#{definition.primary_key}, on_delete: :#{definition.on_delete}) end end diff --git a/scripts/elastic-migration b/scripts/elastic-migration new file mode 100755 index 00000000000..8d7f614160b --- /dev/null +++ b/scripts/elastic-migration @@ -0,0 +1,168 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Generate an Elastic migration file, spec and dictionary record with the current timestamp. + +require 'yaml' +require 'fileutils' +require 'uri' +require 'readline' +require 'active_support/core_ext/string' + +class ElasticMigrationCreator + attr_reader :options + + Options = Struct.new( + :name, + :description, + :group, + :introduced_by_url, + :milestone, + :obsolete, + :marked_obsolete_by_url, + :marked_obsolete_in_milestone + ) + + def initialize + @options = Options.new + end + + def execute + options.name = read_name + options.description = read_description + options.group = read_group + options.introduced_by_url = read_introduced_by_url + options.milestone = read_milestone + + $stdout.puts "\e[32mcreated\e[0m #{file_path}" + $stdout.puts "\e[32mcreated\e[0m #{spec_file_path}" + $stdout.puts "\e[32mcreated\e[0m #{dictionary_file_path}" + + write + $stdout.puts "\n=> Please consult the documentation for Advanced Search Migrations: #{documentation_reference}" + end + + private + + def read_name + read_variable('name', 'Name of the migration in CamelCase').camelize + end + + def read_description + read_variable('description', 'Description of what the migration does') + end + + def read_group + read_variable('group', 'The group introducing a feature flag, like: `global search`') + end + + def read_milestone + milestone = File.read('VERSION') + milestone.gsub(/^(\d+\.\d+).*$/, '\1').chomp + end + + def read_variable(name, description) + $stdout.puts "\n>> #{description}:" + + loop do + variable = Readline.readline('?> ', false)&.strip + return variable unless variable.empty? + + warn "Error: #{name} is required." + end + end + + def read_introduced_by_url + $stdout.puts + $stdout.puts ">> URL of the MR introducing the migration (enter to skip):" + + loop do + introduced_by_url = Readline.readline('?> ', false)&.strip + introduced_by_url = nil if introduced_by_url.empty? + return introduced_by_url if introduced_by_url.nil? || introduced_by_url.start_with?('https://') + + warn 'Error: URL needs to start with https://' + end + end + + def write + # create migration file + FileUtils.mkdir_p(File.dirname(file_path)) + File.write(file_path, file_contents) + + # create spec + FileUtils.mkdir_p(File.dirname(spec_file_path)) + File.write(spec_file_path, spec_contents) + + # create dictionary file + FileUtils.mkdir_p(File.dirname(dictionary_file_path)) + File.write(dictionary_file_path, dictionary_contents) + end + + def timestamp + @timestamp ||= Time.now.strftime('%Y%m%d%H%M%S') + end + + def file_name + @file_name ||= "#{timestamp}_#{options.name.dup.underscore}" + end + + def file_path + "ee/elastic/migrate/#{file_name}.rb" + end + + def spec_file_path + "ee/spec/elastic/migrate/#{file_name}_spec.rb" + end + + def dictionary_file_path + "ee/elastic/docs/#{file_name}.yml" + end + + def file_contents + "# frozen_string_literal: true + +class #{options.name} < Elastic::Migration +end +" + end + + def spec_contents + "# frozen_string_literal: true + +require 'spec_helper' +require_relative 'migration_shared_examples' +require File.expand_path('#{file_path}') + +RSpec.describe #{options.name}, feature_category: :#{options.group.parameterize.underscore} do + let(:version) { #{timestamp} } +end +" + end + + def dictionary_contents + dictionary_config_hash.to_yaml + end + + def dictionary_config_hash + { + 'name' => options.name, + 'version' => timestamp, + 'description' => options.description, + 'group' => "group::#{options.group}", + 'milestone' => options.milestone, + 'introduced_by_url' => options.introduced_by_url, + 'obsolete' => false, + 'marked_obsolete_by_url' => nil, + 'marked_obsolete_in_milestone' => nil + } + end + + def documentation_reference + 'https://docs.gitlab.com/ee/development/search/advanced_search_migration_styleguide.html' + end +end + +ElasticMigrationCreator.new.execute if $PROGRAM_NAME == __FILE__ + +# vim: ft=ruby diff --git a/scripts/frontend/create_jsconfig.js b/scripts/frontend/create_jsconfig.js new file mode 100755 index 00000000000..7e2d0eb02c4 --- /dev/null +++ b/scripts/frontend/create_jsconfig.js @@ -0,0 +1,106 @@ +#!/usr/bin/env node + +/** + * @file This file generates a + * [jsconfig.json](https://code.visualstudio.com/docs/languages/jsconfig) file + * using aliases from the webpack config. To use it run from project root: + * + * ```sh + * node ./scripts/frontend/create_jsconfig.js + * ``` + * + * NOTE: since aliases are currently generated based on solely Webpack config, + * aliases defined in Jest config might be missing. + */ + +const fs = require('node:fs/promises'); +const path = require('node:path'); +const readline = require('node:readline/promises'); +const { stdin, stdout } = require('node:process'); +const chalk = require('chalk').default; +const prettier = require('prettier'); + +const PATH_PROJECT_ROOT = path.resolve(__dirname, '..', '..'); +const PATH_JS_CONFIG = path.join(PATH_PROJECT_ROOT, 'jsconfig.json'); + +/** + * Creates and writes a jsconfig.json file, based on Webpack aliases. + */ +async function createJsConfig() { + // eslint-disable-next-line global-require + const webpackConfig = require('../../config/webpack.config'); + + const paths = {}; // aliases + const WEBPACK_ALIAS_EXCEPTIONS = ['jquery$', '@gitlab/svgs/dist/icons.svg']; + Object.entries(webpackConfig.resolve.alias) + .filter(([key]) => !WEBPACK_ALIAS_EXCEPTIONS.includes(key)) + .forEach(([key, value]) => { + const alias = `${key}/*`; + const target = [`${path.relative(PATH_PROJECT_ROOT, value)}/*`]; + paths[alias] = target; + }); + + const jsConfig = { + // As we're introducing jsconfig to the project, as a precaution we add both: + // 'include' and 'exclude' options. This might be simplified in the future. + exclude: ['node_modules', 'vendor'], + + // 'include' is currently manually defined. We might want to append manually + // defined paths with paths from aliases + include: [ + 'app/assets/javascripts', + 'ee/app/assets/javascripts', + 'spec/frontend', + 'ee/spec/frontend', + 'tmp/tests/frontend/fixtures/', + 'tmp/tests/frontend/fixtures-ee/', + ], + compilerOptions: { + baseUrl: '.', // Define the project root + checkJs: false, // Disable type checking on JavaScript files + disableSizeLimit: true, // Disable memory size limit for the language server + paths, // Aliases + }, + }; + + // Stringify, format and update the config file + const jsConfigString = prettier.format(JSON.stringify(jsConfig, null, 2), { parser: 'json' }); + await fs.writeFile(PATH_JS_CONFIG, jsConfigString); +} + +function fileExists(filePath) { + return fs.stat(filePath).then( + () => true, + () => false, + ); +} + +async function main() { + const jsconfigExists = await fileExists(PATH_JS_CONFIG); + + if (jsconfigExists) { + console.log(`${chalk.yellow('WARNING:')} jsconfig.json file already exists.`); + console.log(''); + const rl = readline.createInterface({ input: stdin, output: stdout }); + const response = await rl.question('Would you like to overwrite it? (y/n) '); + rl.close(); + console.log(''); + + const shouldOverwrite = response.match(/^y(es)?$/i); + + if (!shouldOverwrite) { + console.log('Overwrite cancelled.'); + return; + } + } + + try { + await createJsConfig(); + console.log(chalk.green('jsconfig.json file created.')); + } catch (error) { + console.log(`${chalk.red('ERROR:')} failed to create jsconfig.json. See the error below:`); + console.error(error); + } +} + +main(); diff --git a/scripts/generate-e2e-pipeline b/scripts/generate-e2e-pipeline index 2b882094dc2..31a3122050a 100755 --- a/scripts/generate-e2e-pipeline +++ b/scripts/generate-e2e-pipeline @@ -56,7 +56,11 @@ echo "***Saving generated qa pipeline files***" for key in "${!qa_pipelines[@]}"; do echo "Generating $key" - cp ".gitlab/ci/${qa_pipelines[$key]}" "$key" + if [ "${CI_PROJECT_NAMESPACE}" = "gitlab-cn" ]; then + cp "jh/.gitlab/ci/${qa_pipelines[$key]}" "$key" + else + cp ".gitlab/ci/${qa_pipelines[$key]}" "$key" + fi echo >>"$key" # add empty line so it's easier to read if debugging echo "$variables" >>"$key" diff --git a/scripts/generate-message-to-run-e2e-pipeline.rb b/scripts/generate-message-to-run-e2e-pipeline.rb index af18578add6..cfe480ac5c7 100755 --- a/scripts/generate-message-to-run-e2e-pipeline.rb +++ b/scripts/generate-message-to-run-e2e-pipeline.rb @@ -70,15 +70,20 @@ class GenerateMessageToRunE2ePipeline def content <<~MARKDOWN <!-- Run e2e warning begin --> - :warning: @#{author_username} Some end-to-end (E2E) tests have been selected based on the stage label on this MR. - If not run already, please run the `e2e:package-and-test-ee` job in the `qa` stage - and review the results **before merging this MR**. (E2E tests are not run automatically on some MRs due to [runner resource constraints](https://gitlab.com/gitlab-org/gitlab-qa/-/issues/261).) + @#{author_username} Some end-to-end (E2E) tests should run based on the stage label. - If you would like to run all e2e tests, please apply the ~"pipeline:run-all-e2e" label and restart the pipeline. + Please start the `trigger-omnibus-and-follow-up-e2e` job in the `qa` stage and ensure tests in the `follow-up-e2e:package-and-test-ee` pipeline + pass **before this MR is merged**. + (E2E tests are computationally intensive and don't run automatically for every push/rebase, so we ask you to run this job manually at least once.) - Once done, please apply the ✅ emoji on this comment. + To run all E2E tests, apply the ~"pipeline:run-all-e2e" label and run a new pipeline. - For any questions or help in reviewing the E2E test results, please reach out on the internal #quality Slack channel. + E2E test jobs are allowed to fail due to [flakiness](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/test-metrics-dashboards/#package-and-test). + See current failures at the latest [pipeline triage issue](https://gitlab.com/gitlab-org/quality/pipeline-triage/-/issues). + + Once done, apply the ✅ emoji on this comment. + + For any questions or help, reach out on the internal #quality Slack channel. <!-- Run e2e warning end --> MARKDOWN end diff --git a/scripts/lib/glfm/render_static_html.rb b/scripts/lib/glfm/render_static_html.rb index bf7865c4d95..ef2dc071c38 100644 --- a/scripts/lib/glfm/render_static_html.rb +++ b/scripts/lib/glfm/render_static_html.rb @@ -29,7 +29,6 @@ RSpec.describe 'Render Static HTML', :api, type: :request do include Glfm::Constants include Glfm::Shared - # noinspection RailsParamDefResolve (RubyMine can't find the shared context from this file location) include_context 'with GLFM example snapshot fixtures' it do @@ -49,7 +48,6 @@ RSpec.describe 'Render Static HTML', :api, type: :request do api_url = metadata_hash.dig(name, :api_request_override_path) || (api "/markdown") post api_url, params: { text: markdown, gfm: true } - # noinspection RubyResolve expect(response).to be_successful returned_html_value = diff --git a/scripts/lib/glfm/shared.rb b/scripts/lib/glfm/shared.rb index 56cb2f95d6a..a17f4b30271 100644 --- a/scripts/lib/glfm/shared.rb +++ b/scripts/lib/glfm/shared.rb @@ -29,7 +29,6 @@ module Glfm end def run_external_cmd(cmd) - # noinspection RubyMismatchedArgumentType rails_root = File.expand_path('../../../', __dir__) # See https://stackoverflow.com/a/20001569/25192 diff --git a/scripts/lib/glfm/update_example_snapshots.rb b/scripts/lib/glfm/update_example_snapshots.rb index 8f817d0173e..793f7521283 100644 --- a/scripts/lib/glfm/update_example_snapshots.rb +++ b/scripts/lib/glfm/update_example_snapshots.rb @@ -358,7 +358,6 @@ module Glfm glfm_examples_statuses[name][:already_printed] = true # Copy over the existing example only if it exists and preserve_existing is true, otherwise omit this example - # noinspection RubyScope hash[name] = existing_hash[name] if existing_hash[name] next diff --git a/scripts/lib/glfm/update_specification.rb b/scripts/lib/glfm/update_specification.rb index ef6f24d5a77..0effea83378 100644 --- a/scripts/lib/glfm/update_specification.rb +++ b/scripts/lib/glfm/update_specification.rb @@ -123,7 +123,6 @@ module Glfm validate_expected_spec_version!(ghfm_spec_lines[2]) # Reset IO stream and re-read into a single string for easy writing - # noinspection RubyNilAnalysis ghfm_spec_txt_uri_io.seek(0) ghfm_spec_string = ghfm_spec_txt_uri_io.read raise "Unable to read string from #{GHFM_SPEC_TXT_URI}" unless ghfm_spec_string @@ -268,9 +267,7 @@ module Glfm end # NOTE: body, title, and version are used by the ERB binding. - # noinspection RubyUnusedLocalVariable def add_styling_to_specification_html(body:, title:, version:) - # noinspection RubyMismatchedArgumentType ERB.new(File.read(File.expand_path('specification_html_template.erb', __dir__))).result(binding) end diff --git a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb index ef8872ecbb0..ddfde8eb464 100644 --- a/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb +++ b/scripts/lib/glfm/verify_all_generated_files_are_up_to_date.rb @@ -25,9 +25,7 @@ module Glfm output("Running `yarn install --frozen-lockfile` to ensure `yarn check-dependencies` doesn't fail...") run_external_cmd('yarn install --frozen-lockfile') - # noinspection RubyMismatchedArgumentType update_specification_script = File.expand_path('../../glfm/update-specification.rb', __dir__) - # noinspection RubyMismatchedArgumentType update_example_snapshots_script = File.expand_path('../../glfm/update-example-snapshots.rb', __dir__) output("Running `#{update_specification_script}`...") diff --git a/scripts/license-check.sh b/scripts/license-check.sh index 2a210754a0d..58dc799d0a2 100755 --- a/scripts/license-check.sh +++ b/scripts/license-check.sh @@ -18,6 +18,10 @@ set -euo pipefail PROJECT_PATH=${1:-`pwd`} +function restore_git_state() { + git checkout -q Gemfile Gemfile.lock +} + echo "Using project path ${PROJECT_PATH}" GEMFILE_DIFF=`git diff Gemfile Gemfile.lock` @@ -27,7 +31,7 @@ if [ ! -z "$GEMFILE_DIFF" ]; then exit 1 fi +trap restore_git_state EXIT + BUNDLE_DEPLOYMENT=false BUNDLE_FROZEN=false bundle lock --add-platform `ruby -e "puts RUBY_PLATFORM"` bundle exec license_finder --decisions-file config/dependency_decisions.yml --project-path ${PROJECT_PATH} - -git checkout -q Gemfile Gemfile.lock diff --git a/scripts/merge-auto-explain-logs b/scripts/merge-auto-explain-logs new file mode 100755 index 00000000000..0dff4fb33f8 --- /dev/null +++ b/scripts/merge-auto-explain-logs @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "set" +require "json" + +fingerprints = Set.new + +ARGF.each_line do |line| + fingerprint = JSON.parse(line)['fingerprint'] + $stdout.puts(line) && $stdout.flush if fingerprints.add?(fingerprint) +end + +warn("auto_explain log contains #{fingerprints.size} entries") diff --git a/scripts/merge-simplecov b/scripts/merge-simplecov index 7db12839382..8c816b8048f 100755 --- a/scripts/merge-simplecov +++ b/scripts/merge-simplecov @@ -2,9 +2,10 @@ # frozen_string_literal: true require_relative '../config/bundler_setup' -require_relative '../spec/simplecov_env' -SimpleCovEnv.configure_profile -SimpleCovEnv.configure_formatter +require_relative '../spec/simplecov_env_core' + +SimpleCovEnvCore.configure_profile +SimpleCovEnvCore.configure_formatter resultset_files = Dir.glob(File.join(SimpleCov.coverage_path, '*', '.resultset.json')) exit(0) if resultset_files.empty? diff --git a/scripts/remote_development/run-e2e-tests.sh b/scripts/remote_development/run-e2e-tests.sh index 905bef4754d..63d828d323c 100755 --- a/scripts/remote_development/run-e2e-tests.sh +++ b/scripts/remote_development/run-e2e-tests.sh @@ -22,7 +22,7 @@ export TEST_INSTANCE_URL="${TEST_INSTANCE_URL:-http://gdk.test:3000}" echo "WEBDRIVER_HEADLESS: ${WEBDRIVER_HEADLESS}" echo "QA_SUPER_SIDEBAR_ENABLED: ${QA_SUPER_SIDEBAR_ENABLED}" echo "GITLAB_USERNAME: ${GITLAB_USERNAME}" -echo "DEVFILE_PROJECT: ${AGENT_NAME}" +echo "DEVFILE_PROJECT: ${DEVFILE_PROJECT}" echo "AGENT_NAME: ${AGENT_NAME}" echo "TEST_INSTANCE_URL: ${TEST_INSTANCE_URL}" @@ -33,4 +33,4 @@ working_directory="$(git rev-parse --show-toplevel)/qa" (cd "$working_directory" && \ bundle && \ bundle exec bin/qa Test::Instance::All "$TEST_INSTANCE_URL" -- \ - --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development) + --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development/without_setup) diff --git a/scripts/remote_development/run-smoke-test-suite.sh b/scripts/remote_development/run-smoke-test-suite.sh index 52e7546d47d..9ab692e6a15 100755 --- a/scripts/remote_development/run-smoke-test-suite.sh +++ b/scripts/remote_development/run-smoke-test-suite.sh @@ -31,51 +31,11 @@ printf "${Color_Off}" printf "${BBlue}Running Remote Development backend specs${Color_Off}\n\n" -bin/spring rspec -r spec_helper \ -ee/spec/features/remote_development/workspaces_spec.rb \ -ee/spec/finders/remote_development/workspaces_finder_spec.rb \ -ee/spec/graphql/api/workspace_spec.rb \ +bin/rspec -r spec_helper \ +$(find . -path '**/remote_development/**/*_spec.rb') \ ee/spec/graphql/types/query_type_spec.rb \ -ee/spec/graphql/types/remote_development/workspace_type_spec.rb \ ee/spec/graphql/types/subscription_type_spec.rb \ -ee/spec/lib/remote_development/agent_config/main_integration_spec.rb \ -ee/spec/lib/remote_development/unmatched_result_error_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/authorizer_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/creator_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/devfile_fetcher_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/devfile_flattener_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/editor_component_injector_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/main_integration_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/main_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/post_flatten_devfile_validator_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/pre_flatten_devfile_validator_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/project_cloner_component_injector_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/volume_component_injector_spec.rb \ -ee/spec/lib/remote_development/workspaces/create/volume_definer_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/actual_state_calculator_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/agent_info_parser_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/agent_info_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/desired_config_generator_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/params_parser_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/reconcile_processor_scenarios_spec.rb \ -ee/spec/lib/remote_development/workspaces/reconcile/reconcile_processor_spec.rb \ -ee/spec/lib/remote_development/workspaces/states_spec.rb \ -ee/spec/lib/remote_development/workspaces/update/authorizer_spec.rb \ -ee/spec/lib/remote_development/workspaces/update/main_integration_spec.rb \ -ee/spec/lib/remote_development/workspaces/update/main_spec.rb \ -ee/spec/lib/remote_development/workspaces/update/updater_spec.rb \ -ee/spec/models/remote_development/remote_development_agent_config_spec.rb \ -ee/spec/models/remote_development/workspace_spec.rb \ -ee/spec/requests/api/graphql/mutations/remote_development/workspaces/create_spec.rb \ -ee/spec/requests/api/graphql/mutations/remote_development/workspaces/update_spec.rb \ -ee/spec/requests/api/graphql/remote_development/current_user_workspaces_spec.rb \ -ee/spec/requests/api/graphql/remote_development/workspace_by_id_spec.rb \ -ee/spec/requests/api/graphql/remote_development/workspaces_by_ids_spec.rb \ ee/spec/requests/api/internal/kubernetes_spec.rb \ -ee/spec/services/remote_development/agent_config/update_service_spec.rb \ -ee/spec/services/remote_development/workspaces/create_service_spec.rb \ -ee/spec/services/remote_development/workspaces/reconcile_service_spec.rb \ -ee/spec/services/remote_development/workspaces/update_service_spec.rb \ spec/graphql/types/subscription_type_spec.rb \ spec/lib/result_spec.rb \ spec/support_specs/matchers/result_matchers_spec.rb diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index af6c2ec5383..728763e56d7 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -243,7 +243,7 @@ function download_chart() { function base_config_changed() { if [ -z "${CI_MERGE_REQUEST_IID}" ]; then return; fi - curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq '.changes | any(.old_path == "scripts/review_apps/base-config.yaml")' + curl "${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/changes" | jq --arg path "${GITLAB_REVIEW_APP_BASE_CONFIG_FILE}" '.changes | any(.old_path == $path)' } function parse_gitaly_image_tag() { @@ -263,7 +263,7 @@ function deploy() { local base_config_file_ref="${CI_DEFAULT_BRANCH}" if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi - local base_config_file="${GITLAB_REPO_URL}/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml" + local base_config_file="${GITLAB_REPO_URL}/raw/${base_config_file_ref}/${GITLAB_REVIEW_APP_BASE_CONFIG_FILE}" echoinfo "Deploying ${release} to ${CI_ENVIRONMENT_URL} ..." true @@ -397,7 +397,7 @@ function verify_deploy() { # By default, try for 5 minutes, with 5 of sleep between attempts local max_try_times=$((${GITLAB_VERIFY_DEPLOY_TIMEOUT_MINUTES:-5} * 60 / 5)) - for i in {1..$max_try_times}; do + for i in $(seq 1 $max_try_times); do local now=$(date '+%H:%M:%S') echo "[${now}] Verifying deployment at ${CI_ENVIRONMENT_URL}/users/sign_in" log_name="curl-logs/${now}.log" diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index fd478cbd3ae..cab1ac10f35 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -13,9 +13,9 @@ function retrieve_tests_metadata() { echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" fi - if [[ ! -f "${RSPEC_FAST_QUARANTINE_LOCAL_PATH}" ]]; then - curl --location -o "${RSPEC_FAST_QUARANTINE_LOCAL_PATH}" "https://gitlab-org.gitlab.io/quality/engineering-productivity/fast-quarantine/${RSPEC_FAST_QUARANTINE_LOCAL_PATH}" || - echo "" > "${RSPEC_FAST_QUARANTINE_LOCAL_PATH}" + if [[ ! -f "${RSPEC_FAST_QUARANTINE_PATH}" ]]; then + curl --location -o "${RSPEC_FAST_QUARANTINE_PATH}" "https://gitlab-org.gitlab.io/quality/engineering-productivity/fast-quarantine/${RSPEC_FAST_QUARANTINE_PATH}" || + echo "" > "${RSPEC_FAST_QUARANTINE_PATH}" fi } @@ -179,7 +179,7 @@ function debug_rspec_variables() { echoinfo "FLAKY_RSPEC_SUITE_REPORT_PATH: ${FLAKY_RSPEC_SUITE_REPORT_PATH:-}" echoinfo "FLAKY_RSPEC_REPORT_PATH: ${FLAKY_RSPEC_REPORT_PATH:-}" echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH:-}" - echoinfo "SKIPPED_TESTS_REPORT_PATH: ${SKIPPED_TESTS_REPORT_PATH:-}" + echoinfo "RSPEC_SKIPPED_TESTS_REPORT_PATH: ${RSPEC_SKIPPED_TESTS_REPORT_PATH:-}" echoinfo "CRYSTALBALL: ${CRYSTALBALL:-}" @@ -258,7 +258,6 @@ function rspec_paralellized_job() { export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})") export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json" export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json" - export SKIPPED_TESTS_REPORT_PATH="rspec/skipped_tests_${report_name}.txt" if [[ -d "ee/" ]]; then export KNAPSACK_GENERATE_REPORT="true" @@ -302,13 +301,10 @@ function retry_failed_rspec_examples() { # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE}) local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg13 1/24' would become 'rspec_unit_pg13_1_24' - local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/" - - export RETRIED_TESTS_REPORT_PATH="${rspec_flaky_folder_path}retried_tests_${report_name}_report.txt" - echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}" + echoinfo "RSPEC_RETRIED_TESTS_REPORT_PATH: ${RSPEC_RETRIED_TESTS_REPORT_PATH:-}" - echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}" - echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}" + echo "${CI_JOB_URL}" > "${RSPEC_RETRIED_TESTS_REPORT_PATH:-}" + echo $failed_examples >> "${RSPEC_RETRIED_TESTS_REPORT_PATH:-}" echoinfo "Retrying the failing examples in a new RSpec process..." @@ -357,7 +353,7 @@ function warn_on_successfully_retried_test { # include the root path in the regexp to eliminate false positives changed_file="^\./$changed_file" - if grep -q "$changed_file" "$RETRIED_TESTS_REPORT_PATH"; then + if grep -q "${changed_file}" "${RSPEC_RETRIED_TESTS_REPORT_PATH}"; then echoinfo "Flaky test '$changed_file' was found in the list of files changed by this MR." echoinfo "Exiting with code $SUCCESSFULLY_RETRIED_TEST_EXIT_CODE." exit $SUCCESSFULLY_RETRIED_TEST_EXIT_CODE @@ -454,22 +450,20 @@ function cleanup_individual_job_reports() { rm -rf ${knapsack_folder_path:-unknown_folder}rspec*.json \ ${rspec_flaky_folder_path:-unknown_folder}all_*.json \ ${rspec_flaky_folder_path:-unknown_folder}new_*.json \ - ${rspec_flaky_folder_path:-unknown_folder}skipped_flaky_tests_*_report.txt \ - ${rspec_flaky_folder_path:-unknown_folder}retried_tests_*_report.txt \ + rspec/skipped_flaky_tests_*_report.txt \ + rspec/retried_tests_*_report.txt \ ${RSPEC_LAST_RUN_RESULTS_FILE:-unknown_folder} \ ${RSPEC_PROFILING_FOLDER_PATH:-unknown_folder}/**/* rmdir ${RSPEC_PROFILING_FOLDER_PAT:-unknown_folder} || true } function generate_flaky_tests_reports() { - local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/" - debug_rspec_variables - mkdir -p ${rspec_flaky_folder_path} + mkdir -p rspec/ - find ${rspec_flaky_folder_path} -type f -name 'skipped_tests_*.txt' -exec cat {} + >> "${SKIPPED_TESTS_REPORT_PATH}" - find ${rspec_flaky_folder_path} -type f -name 'retried_tests_*_report.txt' -exec cat {} + >> "${RETRIED_TESTS_REPORT_PATH}" + find rspec/ -type f -name 'skipped_tests-*.txt' -exec cat {} + >> "rspec/skipped_tests_report.txt" + find rspec/ -type f -name 'retried_tests-*.txt' -exec cat {} + >> "rspec/retried_tests_report.txt" cleanup_individual_job_reports } @@ -483,3 +477,13 @@ function is_rspec_last_run_results_file_missing() { return 1 fi } + +function merge_auto_explain_logs() { + local auto_explain_logs_path="$(dirname "${RSPEC_AUTO_EXPLAIN_LOG_PATH}")/" + + for file in ${auto_explain_logs_path}*.gz; do + (gunzip -c "${file}" && rm -f "${file}" || true) + done | \ + scripts/merge-auto-explain-logs | \ + gzip -c > "${RSPEC_AUTO_EXPLAIN_LOG_PATH}" +} diff --git a/scripts/setup-test-env b/scripts/setup-test-env index 1c39483bb7a..50bec46b71a 100755 --- a/scripts/setup-test-env +++ b/scripts/setup-test-env @@ -26,6 +26,7 @@ require 'omniauth' require 'omniauth-github' require 'etc' require 'gitlab/utils/all' +require 'gitlab/safe_request_store' require_relative '../lib/gitlab/access' unless defined?(License) diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh index 445a988b996..ffc3c6582db 100755 --- a/scripts/setup/as-if-jh.sh +++ b/scripts/setup/as-if-jh.sh @@ -19,7 +19,7 @@ download_jh_path() { echoinfo "Downloading ${path}" - curl --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" + curl -f --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" tar -zxf "${output}" --strip-component 1 rm "${output}" diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 89a2bf034cc..8f509399fd4 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -66,7 +66,7 @@ module Trigger def com_gitlab_client @com_gitlab_client ||= Gitlab.client( - endpoint: 'https://gitlab.com/api/v4', + endpoint: endpoint, private_token: self.class.access_token ) end @@ -113,12 +113,16 @@ module Trigger end def stable_branch? - ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ + ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee|-jh)?$/ end def fallback_ref if trigger_stable_branch_if_detected? && stable_branch? - ENV['CI_COMMIT_REF_NAME'].delete_suffix('-ee') + if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn' + ENV['CI_COMMIT_REF_NAME'].delete_suffix('-jh') + elsif ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-org' + ENV['CI_COMMIT_REF_NAME'].delete_suffix('-ee') + end else primary_ref end @@ -142,6 +146,12 @@ module Trigger params[version_file] = version_param_value(version_file) end end + + def endpoint + return "https://jihulab.com/api/v4" if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn' + + "https://gitlab.com/api/v4" + end end class CNG < Base @@ -162,7 +172,9 @@ module Trigger end def primary_ref - 'master' + return "main-jh" if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn' + + "master" end def trigger_stable_branch_if_detected? diff --git a/scripts/utils.sh b/scripts/utils.sh index 800b81f1dea..13a051e2b58 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -357,3 +357,98 @@ function setup_gcloud() { gcloud auth activate-service-account --key-file="${REVIEW_APPS_GCP_CREDENTIALS}" gcloud config set project "${REVIEW_APPS_GCP_PROJECT}" } + +function download_files() { + base_url_prefix="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/files" + base_url_suffix="raw?ref=${CI_COMMIT_SHA}" + + # Construct the list of files to download with curl + for file in "$@"; do + local url_encoded_filename + url_encoded_filename=$(url_encode "${file}") + local file_url="${base_url_prefix}/${url_encoded_filename}/${base_url_suffix}" + echo "url = ${file_url}" >> urls_outputs.txt + echo "output = ${file}" >> urls_outputs.txt + done + + echo "List of files to download:" + cat urls_outputs.txt + + curl -f --header "Private-Token: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" --create-dirs --parallel --config urls_outputs.txt +} + +# Taken from https://gist.github.com/jaytaylor/5a90c49e0976aadfe0726a847ce58736 +# +# It is surprisingly hard to url-encode an URL in shell. shorter alternatives used jq, +# but we would then need to install it wherever we would use this no-clone functionality. +# +# For the purposes of url-encoding filenames, this function should be enough. +function url_encode() { + echo "$@" | sed \ + -e 's/%/%25/g' \ + -e 's/ /%20/g' \ + -e 's/!/%21/g' \ + -e 's/"/%22/g' \ + -e "s/'/%27/g" \ + -e 's/#/%23/g' \ + -e 's/(/%28/g' \ + -e 's/)/%29/g' \ + -e 's/+/%2b/g' \ + -e 's/,/%2c/g' \ + -e 's/-/%2d/g' \ + -e 's/:/%3a/g' \ + -e 's/;/%3b/g' \ + -e 's/?/%3f/g' \ + -e 's/@/%40/g' \ + -e 's/\$/%24/g' \ + -e 's/\&/%26/g' \ + -e 's/\*/%2a/g' \ + -e 's/\./%2e/g' \ + -e 's/\//%2f/g' \ + -e 's/\[/%5b/g' \ + -e 's/\\/%5c/g' \ + -e 's/\]/%5d/g' \ + -e 's/\^/%5e/g' \ + -e 's/_/%5f/g' \ + -e 's/`/%60/g' \ + -e 's/{/%7b/g' \ + -e 's/|/%7c/g' \ + -e 's/}/%7d/g' \ + -e 's/~/%7e/g' +} + +# Download the local gems in `gems` and `vendor/gems` folders from the API. +# +# This is useful if you need to run bundle install while not doing a git clone of the gitlab-org/gitlab repo. +function download_local_gems() { + for folder_path in vendor/gems gems; do + local output="${folder_path}.tar.gz" + + # From https://docs.gitlab.com/ee/api/repositories.html#get-file-archive: + # + # This endpoint can be accessed without authentication if the repository is publicly accessible. + # For GitLab.com users, this endpoint has a rate limit threshold of 5 requests per minute. + # + # We don't want to set a token for public repo (e.g. gitlab-org/gitlab), as 5 requests/minute can + # potentially be reached with many pipelines running in parallel. + local private_token_header="" + if [[ "${CI_PROJECT_VISIBILITY}" != "public" ]]; then + private_token_header="Private-Token: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" + fi + + echo "Downloading ${folder_path}" + + url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/archive" + curl -f \ + --create-dirs \ + --get \ + --header "${private_token_header}" \ + --output "${output}" \ + --data-urlencode "sha=${CI_COMMIT_SHA}" \ + --data-urlencode "path=${folder_path}" \ + "${url}" + + tar -zxf "${output}" --strip-component 1 + rm "${output}" + done +} diff --git a/scripts/validate_name_collisions_between_migrations b/scripts/validate_name_collisions_between_migrations new file mode 100755 index 00000000000..b6bc46cf480 --- /dev/null +++ b/scripts/validate_name_collisions_between_migrations @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +require_relative './database/migration_collision_checker' + +result = MigrationCollisionChecker.new.check + +if result + puts result.error_message + exit result.error_code +end |