diff options
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/generate-memory-metrics-on-boot | 29 | ||||
-rw-r--r-- | scripts/gitlab_workhorse_component_helpers.sh | 73 | ||||
-rwxr-xr-x | scripts/ingest-reports-to-siem | 45 | ||||
-rwxr-xr-x | scripts/lint_templates_bash.rb | 76 | ||||
-rwxr-xr-x | scripts/merge-simplecov | 24 | ||||
-rwxr-xr-x | scripts/setup/find-jh-branch.rb | 2 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 4 | ||||
-rw-r--r-- | scripts/utils.sh | 16 |
8 files changed, 229 insertions, 40 deletions
diff --git a/scripts/generate-memory-metrics-on-boot b/scripts/generate-memory-metrics-on-boot index 945661aa057..539446f7c0c 100755 --- a/scripts/generate-memory-metrics-on-boot +++ b/scripts/generate-memory-metrics-on-boot @@ -1,12 +1,29 @@ #!/usr/bin/env ruby # frozen_string_literal: true -abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1 -memory_bundle_mem_file_name = ARGV.first +abort "usage: #{__FILE__} <memory_bundle_mem_file_name_prefix> <test_count>" unless ARGV.length == 2 +memory_bundle_mem_file_name_prefix = ARGV.first +test_count = ARGV.last.to_i -full_report = File.open(memory_bundle_mem_file_name).read +results = [] +(1..test_count).each do |i| + report_filename = "#{memory_bundle_mem_file_name_prefix}#{i}.txt" -stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report) -abort 'failed to process the benchmark output' unless stats + stats = nil + File.foreach(report_filename).detect do |line| + stats = /TOP: (?<total_mibs_str>.*) MiB/.match(line) + end + abort 'failed to process the benchmark output' unless stats -puts "total_memory_used_by_dependencies_on_boot_prod_env_mb #{stats[:total_mibs_str].to_f.round(1)}" + total_mibs = stats[:total_mibs_str].to_f + results << total_mibs +end + +res = results.sort +median = (res[(test_count - 1) / 2] + res[test_count / 2]) / 2.0 + +METRIC_NAME = "total_memory_used_by_dependencies_on_boot_prod_env_mb" + +puts "# TYPE #{METRIC_NAME} gauge" +puts "# UNIT #{METRIC_NAME} mebibytes" +puts "#{METRIC_NAME} #{median.round(1)}" diff --git a/scripts/gitlab_workhorse_component_helpers.sh b/scripts/gitlab_workhorse_component_helpers.sh new file mode 100644 index 00000000000..06fe7b2ea51 --- /dev/null +++ b/scripts/gitlab_workhorse_component_helpers.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export CURL_TOKEN_HEADER="${CURL_TOKEN_HEADER:-"JOB-TOKEN"}" +export GITLAB_WORKHORSE_BINARIES_LIST="gitlab-resize-image gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse" +export GITLAB_WORKHORSE_PACKAGE_FILES_LIST="${GITLAB_WORKHORSE_BINARIES_LIST} WORKHORSE_TREE" +export GITLAB_WORKHORSE_TREE=${GITLAB_WORKHORSE_TREE:-$(git rev-parse HEAD:workhorse)} +export GITLAB_WORKHORSE_PACKAGE="workhorse-${GITLAB_WORKHORSE_TREE}.tar.gz" +export GITLAB_WORKHORSE_PACKAGE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${GITLAB_WORKHORSE_FOLDER}/${GITLAB_WORKHORSE_TREE}/${GITLAB_WORKHORSE_PACKAGE}" + +function gitlab_workhorse_archive_doesnt_exist() { + local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}" + + status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}") + + [[ "${status}" != "200" ]] +} + +function create_gitlab_workhorse_package() { + local archive_filename="${GITLAB_WORKHORSE_PACKAGE}" + local folder_to_archive="${GITLAB_WORKHORSE_FOLDER}" + local workhorse_folder_path="${TMP_TEST_GITLAB_WORKHORSE_PATH}" + local tar_working_folder="${TMP_TEST_FOLDER}" + + echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${folder_to_archive}'" + tar -czvf ${archive_filename} -C ${tar_working_folder} ${folder_to_archive} + du -h ${archive_filename} +} + +function extract_gitlab_workhorse_package() { + local tar_working_folder="${TMP_TEST_FOLDER}" + + echoinfo "Extracting archive to ${tar_working_folder}" + + tar -xzv -C ${tar_working_folder} < /dev/stdin +} + +function upload_gitlab_workhorse_package() { + local archive_filename="${GITLAB_WORKHORSE_PACKAGE}" + local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}" + local token_header="${CURL_TOKEN_HEADER}" + local token="${CI_JOB_TOKEN}" + + echoinfo "Uploading ${archive_filename} to ${package_url} ..." + curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}" +} + +function read_curl_gitlab_workhorse_package() { + local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}" + local token_header="${CURL_TOKEN_HEADER}" + local token="${CI_JOB_TOKEN}" + + echoinfo "Downloading from ${package_url} ..." + + curl --fail --silent --retry 3 --header "${token_header}: ${token}" "${package_url}" +} + +function download_and_extract_gitlab_workhorse_package() { + read_curl_gitlab_workhorse_package | extract_gitlab_workhorse_package +} + +function select_gitlab_workhorse_essentials() { + local tmp_path="${CI_PROJECT_DIR}/tmp/${GITLAB_WORKHORSE_FOLDER}" + local original_gitlab_workhorse_path="${TMP_TEST_GITLAB_WORKHORSE_PATH}" + + mkdir -p ${tmp_path} + cd ${original_gitlab_workhorse_path} && mv ${GITLAB_WORKHORSE_PACKAGE_FILES_LIST} ${tmp_path} && cd - + rm -rf ${original_gitlab_workhorse_path} + + # Move the temp folder to its final destination + mv ${tmp_path} ${TMP_TEST_FOLDER} +} diff --git a/scripts/ingest-reports-to-siem b/scripts/ingest-reports-to-siem new file mode 100755 index 00000000000..86c72e1d7eb --- /dev/null +++ b/scripts/ingest-reports-to-siem @@ -0,0 +1,45 @@ +#!/usr/bin/env node + +const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3') +const { fromIni } = require('@aws-sdk/credential-provider-ini') +const path = require('path') +const fs = require('fs') +const crypto = require('crypto') + +function getMD5HashFromFile(data) { + const hash = crypto.createHash('md5').update(data).digest('base64') + return hash +} + +(async function () { + const s3Client = new S3Client({ + region: 'us-east-2', + credentials: fromIni({ profile: 'gl-logs-for-panther' }), + }) + try { + const file = 'gl-dependency-scanning-report.json' + const data = fs.readFileSync(file) + + const [filename, fileext] = path.basename(file).split('.') + const uniqueId = process.env['CI_PIPELINE_ID'] && process.env['CI_JOB_ID'] ? + process.env['CI_PIPELINE_ID'] + '-' + process.env['CI_JOB_ID'] : + Date.now() + const key = path.join('package_hunter_test', filename + '-' + uniqueId + '.' + fileext) + + const responseData = await s3Client.send( + new PutObjectCommand({ + Bucket: 'gl-logs-for-panther-test', + Key: key, + Body: data, + ContentMD5: getMD5HashFromFile(data), + }), + ) + console.log('Successfully uploaded %s to %s', file, key) + } catch (err) { + if (err.name === 'CredentialsProviderError' || err.name === 'AuthorizationHeaderMalformed') + console.log('Could not upload the report. Are AWS credentials configured in ~/.aws/credentials?') + else + console.log('Unexpected error during upload: ', err.message) + process.exit(1) + } +})() diff --git a/scripts/lint_templates_bash.rb b/scripts/lint_templates_bash.rb new file mode 100755 index 00000000000..8db9469ecdf --- /dev/null +++ b/scripts/lint_templates_bash.rb @@ -0,0 +1,76 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../config/environment' +require 'open3' + +module LintTemplatesBash + module_function + + EXCLUDED_RULES = [ + "SC2046", "SC2086", # will be fixed later: https://gitlab.com/gitlab-org/gitlab/-/issues/352973 + "SC1090", "SC1091", # we do not have access to sourced files for analysis. + "SC2154", # Referencing undefined variables is common and adding per-line exceptions for them is unintuitive for end-users + "SC2164" # CI/CD automatically fails if attempting to change to a directory which does not exist. + ].join(",").freeze + + EXCLUDED_TEMPLATES = [ + "dotNET.gitlab-ci.yml" # Powershell + ].freeze + + def run + failed_templates = Gitlab::Template::GitlabCiYmlTemplate.all.filter_map do |template| + next if EXCLUDED_TEMPLATES.include?(template.full_name) + + success = check_template(template) + + template.full_name unless success + end + + if failed_templates.any? + puts "The following templates have shellcheck violations:" + puts failed_templates.join("\n") + exit 1 + end + end + + def process_content(content) + Gitlab::Ci::YamlProcessor.new(content).execute + end + + def job_script(job) + parts = [:before_script, :script, :after_script].map do |key| + job[key]&.join("\n") + end.compact + + parts.prepend("#!/bin/bash\n").join("\n") + end + + def shellcheck(script_content) + combined_streams, status = Open3.capture2e("shellcheck --exclude='#{EXCLUDED_RULES}' -", stdin_data: script_content) + + [combined_streams, status.success?] + end + + def check_job(job) + shellcheck(job_script(job)) + end + + def check_template(template) + parsed = process_content(template.content) + results = parsed.jobs.map do |name, job| + out, success = check_job(job) + + unless success + puts "The '#{name}' job in #{template.full_name} has shellcheck failures:" + puts out + end + + success + end + + results.all? + end +end + +LintTemplatesBash.run diff --git a/scripts/merge-simplecov b/scripts/merge-simplecov index b74a416a6e0..32a0cd86f82 100755 --- a/scripts/merge-simplecov +++ b/scripts/merge-simplecov @@ -5,26 +5,4 @@ require_relative '../spec/simplecov_env' SimpleCovEnv.configure_profile SimpleCovEnv.configure_formatter -module SimpleCov - module ResultMerger - class << self - def resultset_files - Dir.glob(File.join(SimpleCov.coverage_path, '*', '.resultset.json')) - end - - def resultset_hashes - resultset_files.map do |path| - JSON.parse(File.read(path)) - rescue StandardError - {} - end - end - - def resultset - resultset_hashes.reduce({}, :merge) - end - end - end -end - -SimpleCov::ResultMerger.merged_result.format! +SimpleCov.collate Dir.glob(File.join(SimpleCov.coverage_path, '*', '.resultset.json')) diff --git a/scripts/setup/find-jh-branch.rb b/scripts/setup/find-jh-branch.rb index 89aa1492939..a7c1cafd74c 100755 --- a/scripts/setup/find-jh-branch.rb +++ b/scripts/setup/find-jh-branch.rb @@ -8,7 +8,7 @@ require_relative '../api/default_options' class FindJhBranch JH_DEFAULT_BRANCH = 'main-jh' - JH_PROJECT_PATH = 'gitlab-org/gitlab-jh/gitlab' + JH_PROJECT_PATH = 'gitlab-org/gitlab-jh-mirrors/gitlab' BranchNotFound = Class.new(RuntimeError) def run diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 17cbd91a8ee..a3356c664d1 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -324,8 +324,8 @@ module Trigger def invoke!(post_comment: false, downstream_job_name: nil) pipeline = super gitlab = gitlab_client(:upstream) - project_path = base_variables['TOP_UPSTREAM_SOURCE_PROJECT'] - merge_request_id = base_variables['TOP_UPSTREAM_MERGE_REQUEST_IID'] + project_path = variables['TOP_UPSTREAM_SOURCE_PROJECT'] + merge_request_id = variables['TOP_UPSTREAM_MERGE_REQUEST_IID'] comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \ "(limited access). This comment will be updated once the pipeline has finished running." diff --git a/scripts/utils.sh b/scripts/utils.sh index c20508617b8..e896fe40e06 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -83,7 +83,7 @@ function install_junit_merge_gem() { function run_timed_command() { local cmd="${1}" - local metric_name="${2}" + local metric_name="${2:-no}" local timed_metric_file local start=$(date +%s) @@ -97,7 +97,7 @@ function run_timed_command() { if [[ $ret -eq 0 ]]; then echosuccess "==> '${cmd}' succeeded in ${runtime} seconds." - if [[ -n "${metric_name}" ]]; then + if [[ "${metric_name}" != "no" ]]; then timed_metric_file=$(timed_metric_file $metric_name) echo "# TYPE ${metric_name} gauge" > "${timed_metric_file}" echo "# UNIT ${metric_name} seconds" >> "${timed_metric_file}" @@ -132,9 +132,9 @@ function timed_metric_file() { } function echoerr() { - local header="${2}" + local header="${2:-no}" - if [ -n "${header}" ]; then + if [ "${header}" != "no" ]; then printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; else printf "\033[0;31m%s\n\033[0m" "${1}" >&2; @@ -142,9 +142,9 @@ function echoerr() { } function echoinfo() { - local header="${2}" + local header="${2:-no}" - if [ -n "${header}" ]; then + if [ "${header}" != "no" ]; then printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; else printf "\033[0;33m%s\n\033[0m" "${1}" >&2; @@ -152,9 +152,9 @@ function echoinfo() { } function echosuccess() { - local header="${2}" + local header="${2:-no}" - if [ -n "${header}" ]; then + if [ "${header}" != "no" ]; then printf "\n\033[0;32m** %s **\n\033[0m" "${1}" >&2; else printf "\033[0;32m%s\n\033[0m" "${1}" >&2; |