diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 12:07:57 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-20 12:07:57 +0300 |
commit | 7881eb30eaa8b01dbcfe87faa09927c75c7d6e45 (patch) | |
tree | 298bc8d2c62b2f2c29cb8ecbcf3de3eaaa6466d9 /scripts | |
parent | 64b66e0cb6d1bfd27abf24e06653f00bddb60597 (diff) |
Add latest changes from gitlab-org/gitlab@12-6-stable-ee
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/ee-specific-lines-check | 42 | ||||
-rwxr-xr-x | scripts/frontend/check_no_partial_karma_jest.sh | 44 | ||||
-rwxr-xr-x | scripts/get-job-id | 43 | ||||
-rw-r--r-- | scripts/gitaly_test.rb | 8 | ||||
-rwxr-xr-x | scripts/lint-conflicts.sh | 2 | ||||
-rwxr-xr-x | scripts/notifications.sh | 27 | ||||
-rw-r--r-- | scripts/prepare_build.sh | 34 | ||||
-rwxr-xr-x | scripts/review_apps/automated_cleanup.rb | 19 | ||||
-rw-r--r-- | scripts/review_apps/base-config.yaml | 26 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 216 | ||||
-rwxr-xr-x | scripts/security-harness | 116 | ||||
-rwxr-xr-x | scripts/static-analysis | 3 | ||||
-rw-r--r-- | scripts/sync-stable-branch.sh | 14 | ||||
-rwxr-xr-x | scripts/trigger-build | 2 | ||||
-rw-r--r-- | scripts/utils.sh | 13 |
15 files changed, 396 insertions, 213 deletions
diff --git a/scripts/ee-specific-lines-check b/scripts/ee-specific-lines-check deleted file mode 100755 index 4114575168c..00000000000 --- a/scripts/ee-specific-lines-check +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env ruby - -require_relative 'ee_specific_check/ee_specific_check' - -include EESpecificCheck # rubocop:disable Style/MixinUsage -git_version - -base = find_compare_base - -current_numstat = updated_diff_numstat(base.ce_base, base.ee_base) -updated_numstat = updated_diff_numstat(base.ce_head, base.ee_head) - -offenses = updated_numstat.select do |file, updated_delta| - current_delta = current_numstat[file] - - more_lines = updated_delta > current_delta - - more_lines && - !WHITELIST.any? { |pattern| Dir.glob(pattern, File::FNM_DOTMATCH).include?(file) } -end - -if offenses.empty? - say "🎉 All good, congrats! 🎉" -else - puts - - offenses.each do |(file, delta)| - puts "* 💥 #{file} has #{delta - current_numstat[file]} updated lines that differ between EE and CE! 💥" - end - - say <<~MESSAGE - ℹ️ Make sure all lines in shared files have been updated in your backport merge request and the branch name includes #{minimal_ce_branch_name}. - ℹ️ Consider using an EE module to add the features you want. - ℹ️ See this for detail: https://docs.gitlab.com/ee/development/ee_features.html#ee-features-based-on-ce-features - MESSAGE -end - -remove_remotes - -say "ℹ️ For more information on why, see https://gitlab.com/gitlab-org/gitlab/issues/2952" - -exit(offenses.size) diff --git a/scripts/frontend/check_no_partial_karma_jest.sh b/scripts/frontend/check_no_partial_karma_jest.sh new file mode 100755 index 00000000000..c5fffa5900b --- /dev/null +++ b/scripts/frontend/check_no_partial_karma_jest.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +karma_directory=spec/javascripts + +if [ -d ee ]; then + karma_directory="$karma_directory ee/$karma_directory" +fi + +karma_files=$(find $karma_directory -type f -name '*_spec.js' -not -path '*/helpers/*') +violations="" + +for karma_file in $karma_files; do + jest_file=${karma_file/spec\/javascripts/"spec/frontend"} + + if [ -f $jest_file ]; then + violations="$violations $jest_file" + fi +done + +if [[ -z "$violations" ]]; then + echo "All good!" + exit 0 +else + echo "Danger! The following Jest specs have corresponding files in the Karma spec directory (i.e. spec/javascripts):" + echo "" + echo "------------------------------" + for file in $violations; do + echo $file + done + echo "------------------------------" + echo "" + echo "For each of these files, please either:" + echo "" + echo "1. Fully migrate the file to Jest and remove the corresponding Karma file." + echo "2. Remove the Jest file for now, make any relevant changes in the corresponding Karma file, and handle the migration to Jest in a separate MR." + echo "" + echo "Why is this a problem?" + echo "" + echo "- It's nice to have a single source of truth for the unit tests of a subject." + echo "- This will cause conflicts if the remaining Karma spec is migrated using our automated tool." + echo " https://gitlab.com/gitlab-org/frontend/playground/migrate-karma-to-jest" + echo "" + exit 1 +fi diff --git a/scripts/get-job-id b/scripts/get-job-id new file mode 100755 index 00000000000..a5d34dc545b --- /dev/null +++ b/scripts/get-job-id @@ -0,0 +1,43 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab' +require 'optparse' + +# +# Configure credentials to be used with gitlab gem +# +Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + config.private_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'] +end + +options = {} +OptionParser.new do |opts| + opts.on("-s", "--scope=SCOPE", "Find job with matching scope") do |scope| + options[:scope] = scope + end +end.parse! + +class PipelineJobFinder + def initialize(project_id, pipeline_id, job_name, options) + @project_id = project_id + @pipeline_id = pipeline_id + @job_name = job_name + @options = options + end + + def execute + Gitlab.pipeline_jobs(@project_id, @pipeline_id, @options).auto_paginate do |job| + break job if job.name == @job_name + end + end +end + +project_id, pipeline_id, job_name = ARGV + +job = PipelineJobFinder.new(project_id, pipeline_id, job_name, options).execute + +return if job.nil? + +puts job.id diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index b5cc5118530..922dc17ed2e 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -56,6 +56,14 @@ module GitalyTest end def check_gitaly_config! + puts "Checking gitaly-ruby Gemfile..." + + unless File.exist?(gemfile) + message = "#{gemfile} does not exist." + message += "\n\nThis might have happened if the CI artifacts for this build were destroyed." if ENV['CI'] + abort message + end + puts 'Checking gitaly-ruby bundle...' abort 'bundle check failed' unless system(env, 'bundle', 'check', chdir: File.dirname(gemfile)) end diff --git a/scripts/lint-conflicts.sh b/scripts/lint-conflicts.sh index f3877600c8c..2d04507f81e 100755 --- a/scripts/lint-conflicts.sh +++ b/scripts/lint-conflicts.sh @@ -1,5 +1,5 @@ #!/bin/sh -output=`git ls-files -z | grep -zvE '\.(rb|js|haml)$' | xargs -0n1 grep -HEn '^<<<<<<< '` +output=`git grep -En '^<<<<<<< ' -- . ':(exclude)spec/lib/gitlab/conflict/file_spec.rb' ':(exclude)spec/lib/gitlab/git/conflict/parser_spec.rb'` echo $output test -z "$output" diff --git a/scripts/notifications.sh b/scripts/notifications.sh new file mode 100755 index 00000000000..d1b11d44e88 --- /dev/null +++ b/scripts/notifications.sh @@ -0,0 +1,27 @@ +# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set). +# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it). +function notify_slack() { + CHANNEL=$1 + MSG=$2 + ICON_EMOJI=$3 + + if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then + echo "Missing argument(s) - Use: $0 channel message icon_emoji" + echo "and set CI_SLACK_WEBHOOK_URL environment variable." + else + curl -X POST --data-urlencode 'payload={"channel": "#'"${CHANNEL}"'", "username": "GitLab QA Bot", "text": "'"${MSG}"'", "icon_emoji": "'":${ICON_EMOJI}:"'"}' "${CI_SLACK_WEBHOOK_URL}" + fi +} + +function notify_on_job_failure() { + JOB_NAME=$1 + CHANNEL=$2 + MSG=$3 + ICON_EMOJI=$4 + + local job_id + job_id=$(scripts/get-job-id "$CI_PROJECT_ID" "$CI_PIPELINE_ID" "$JOB_NAME" -s failed) + if [ -n "${job_id}" ]; then + notify_slack "${CHANNEL}" "${MSG}" "${ICON_EMOJI}" + fi +} diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 0950ec272a5..e80d752f09f 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -16,37 +16,21 @@ retry gem install knapsack --no-document cp config/gitlab.yml.example config/gitlab.yml sed -i 's/bin_path: \/usr\/bin\/git/bin_path: \/usr\/local\/bin\/git/' config/gitlab.yml -# Determine the database by looking at the job name. -# This would make the default database postgresql. -if [[ "${CI_JOB_NAME#*mysql}" != "$CI_JOB_NAME" ]]; then - export GITLAB_DATABASE='mysql' -else - export GITLAB_DATABASE='postgresql' -fi - -cp config/database.yml.$GITLAB_DATABASE config/database.yml +cp config/database.yml.postgresql config/database.yml -if [ -f config/database_geo.yml.$GITLAB_DATABASE ]; then - cp config/database_geo.yml.$GITLAB_DATABASE config/database_geo.yml +if [ -f config/database_geo.yml.postgresql ]; then + cp config/database_geo.yml.postgresql config/database_geo.yml fi # Set user to a non-superuser to ensure we test permissions sed -i 's/username: root/username: gitlab/g' config/database.yml -if [ "$GITLAB_DATABASE" = 'postgresql' ]; then - sed -i 's/localhost/postgres/g' config/database.yml - sed -i 's/username: git/username: postgres/g' config/database.yml - - if [ -f config/database_geo.yml ]; then - sed -i 's/localhost/postgres/g' config/database_geo.yml - sed -i 's/username: git/username: postgres/g' config/database_geo.yml - fi -else # Assume it's mysql - sed -i 's/localhost/mysql/g' config/database.yml +sed -i 's/localhost/postgres/g' config/database.yml +sed -i 's/username: git/username: postgres/g' config/database.yml - if [ -f config/database_geo.yml ]; then - sed -i 's/localhost/mysql/g' config/database_geo.yml - fi +if [ -f config/database_geo.yml ]; then + sed -i 's/localhost/postgres/g' config/database_geo.yml + sed -i 's/username: git/username: postgres/g' config/database_geo.yml fi cp config/resque.yml.example config/resque.yml @@ -63,6 +47,6 @@ sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml if [ "$SETUP_DB" != "false" ]; then setup_db -elif getent hosts postgres || getent hosts mysql; then +elif getent hosts postgres; then setup_db_user_only fi diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index c7ab8829088..8a04d8e00bc 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -25,7 +25,6 @@ class AutomatedCleanup def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN']) @project_path = project_path @gitlab_token = gitlab_token - ENV['TILLER_NAMESPACE'] ||= review_apps_namespace end def gitlab @@ -45,7 +44,9 @@ class AutomatedCleanup end def helm - @helm ||= Quality::HelmClient.new(namespace: review_apps_namespace) + @helm ||= Quality::HelmClient.new( + tiller_namespace: review_apps_namespace, + namespace: review_apps_namespace) end def kubernetes @@ -75,9 +76,11 @@ class AutomatedCleanup deployed_at = Time.parse(last_deploy) if deployed_at < delete_threshold - delete_environment(environment, deployment) - release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) - releases_to_delete << release + deleted_environment = delete_environment(environment, deployment) + if deleted_environment + release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) + releases_to_delete << release + end elsif deployed_at < stop_threshold stop_environment(environment, deployment) else @@ -116,11 +119,17 @@ class AutomatedCleanup def delete_environment(environment, deployment) print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'deleting') gitlab.delete_environment(project_path, environment.id) + + rescue Gitlab::Error::Forbidden + puts "Review app '#{environment.slug}' is forbidden: skipping it" end def stop_environment(environment, deployment) print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping') gitlab.stop_environment(project_path, environment.id) + + rescue Gitlab::Error::Forbidden + puts "Review app '#{environment.slug}' is forbidden: skipping it" end def helm_releases diff --git a/scripts/review_apps/base-config.yaml b/scripts/review_apps/base-config.yaml index 7aaa7544c19..999f730efc5 100644 --- a/scripts/review_apps/base-config.yaml +++ b/scripts/review_apps/base-config.yaml @@ -8,6 +8,8 @@ global: configureCertmanager: false tls: secretName: tls-cert + initialRootPassword: + secret: shared-gitlab-initial-root-password certmanager: install: false gitlab: @@ -42,7 +44,7 @@ gitlab: memory: 37.5M maxReplicas: 3 hpa: - targetAverageValue: 130m + targetAverageValue: 500m deployment: livenessProbe: timeoutSeconds: 5 @@ -50,10 +52,12 @@ gitlab: resources: requests: cpu: 650m - memory: 880M + memory: 970M limits: cpu: 975m - memory: 1320M + memory: 1450M + hpa: + targetAverageValue: 650m task-runner: resources: requests: @@ -66,10 +70,10 @@ gitlab: resources: requests: cpu: 500m - memory: 1540M + memory: 1630M limits: cpu: 750m - memory: 2310M + memory: 2450M deployment: readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -78,10 +82,10 @@ gitlab: workhorse: resources: requests: - cpu: 250m + cpu: 400m memory: 50M limits: - cpu: 375m + cpu: 600m memory: 75M readinessProbe: initialDelaySeconds: 5 # Default is 0 @@ -90,18 +94,18 @@ gitlab: gitlab-runner: resources: requests: - cpu: 450m + cpu: 675m memory: 100M limits: - cpu: 675m + cpu: 1015m memory: 150M minio: resources: requests: - cpu: 5m + cpu: 9m memory: 128M limits: - cpu: 10m + cpu: 15m memory: 280M nginx-ingress: controller: diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index ed872783856..62360dfe298 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -2,29 +2,31 @@ function deploy_exists() { local namespace="${1}" - local deploy="${2}" - echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true + local release="${2}" + local deploy_exists - helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1 - local deploy_exists=$? + echoinfo "Checking if ${release} exists in the ${namespace} namespace..." true - echoinfo "Deployment status for ${deploy} is ${deploy_exists}" + helm status --tiller-namespace "${namespace}" "${release}" >/dev/null 2>&1 + deploy_exists=$? + + echoinfo "Deployment status for ${release} is ${deploy_exists}" return $deploy_exists } function previous_deploy_failed() { local namespace="${1}" - local deploy="${2}" + local release="${2}" - echoinfo "Checking for previous deployment of ${deploy}" true + echoinfo "Checking for previous deployment of ${release}" true - helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1 + helm status --tiller-namespace "${namespace}" "${release}" >/dev/null 2>&1 local status=$? # if `status` is `0`, deployment exists, has a status if [ $status -eq 0 ]; then echoinfo "Previous deployment found, checking status..." - deployment_status=$(helm status --tiller-namespace "${namespace}" "${deploy}" | grep ^STATUS | cut -d' ' -f2) + deployment_status=$(helm status --tiller-namespace "${namespace}" "${release}" | grep ^STATUS | cut -d' ' -f2) echoinfo "Previous deployment state: ${deployment_status}" if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then status=0; @@ -39,30 +41,53 @@ function previous_deploy_failed() { function delete_release() { local namespace="${KUBE_NAMESPACE}" - local deploy="${CI_ENVIRONMENT_SLUG}" + local release="${CI_ENVIRONMENT_SLUG}" - if [ -z "$deploy" ]; then + if [ -z "${release}" ]; then echoerr "No release given, aborting the delete!" return fi - echoinfo "Deleting release '$deploy'..." true + helm_delete_release "${namespace}" "${release}" + kubectl_cleanup_release "${namespace}" "${release}" +} + +function helm_delete_release() { + local namespace="${1}" + local release="${2}" + + echoinfo "Deleting Helm release '${release}'..." true + + helm delete --tiller-namespace "${namespace}" --purge "${release}" +} - helm delete --purge --tiller-namespace "${namespace}" "${deploy}" +function kubectl_cleanup_release() { + local namespace="${1}" + local release="${2}" + + echoinfo "Deleting all K8s resources matching '${release}'..." true + kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \ + | grep "${release}" \ + | awk '{print $1}' \ + | xargs kubectl --namespace "${namespace}" delete \ + || true } function delete_failed_release() { - if [ -z "$CI_ENVIRONMENT_SLUG" ]; then + local namespace="${KUBE_NAMESPACE}" + local release="${CI_ENVIRONMENT_SLUG}" + + if [ -z "${release}" ]; then echoerr "No release given, aborting the delete!" return fi - if ! deploy_exists "${KUBE_NAMESPACE}" "${CI_ENVIRONMENT_SLUG}"; then - echoinfo "No Review App with ${CI_ENVIRONMENT_SLUG} is currently deployed." + if ! deploy_exists "${namespace}" "${release}"; then + echoinfo "No Review App with ${release} is currently deployed." else # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade` - if previous_deploy_failed "${KUBE_NAMESPACE}" "$CI_ENVIRONMENT_SLUG" ; then - echoinfo "Review App deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG" + if previous_deploy_failed "${namespace}" "${release}" ; then + echoinfo "Review App deployment in bad state, cleaning up ${release}" delete_release else echoinfo "Review App deployment in good state" @@ -72,9 +97,12 @@ function delete_failed_release() { function get_pod() { + local namespace="${KUBE_NAMESPACE}" + local release="${CI_ENVIRONMENT_SLUG}" local app_name="${1}" local status="${2-Running}" - get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1" + + get_pod_cmd="kubectl get pods --namespace ${namespace} --field-selector=status.phase=${status} -lapp=${app_name},release=${release} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1" echoinfo "Waiting till '${app_name}' pod is ready" true echoinfo "Running '${get_pod_cmd}'" @@ -113,58 +141,68 @@ function check_kube_domain() { } function ensure_namespace() { - echoinfo "Ensuring the ${KUBE_NAMESPACE} namespace exists..." true + local namespace="${KUBE_NAMESPACE}" + + echoinfo "Ensuring the ${namespace} namespace exists..." true - kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + kubectl describe namespace "${namespace}" || kubectl create namespace "${namespace}" } function install_tiller() { - local TILLER_NAMESPACE="$KUBE_NAMESPACE" - echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true + local namespace="${KUBE_NAMESPACE}" + + echoinfo "Checking deployment/tiller-deploy status in the ${namespace} namespace..." true echoinfo "Initiating the Helm client..." helm init --client-only # Set toleration for Tiller to be installed on a specific node pool helm init \ + --tiller-namespace "${namespace}" \ --wait \ --upgrade \ + --force-upgrade \ --node-selectors "app=helm" \ --replicas 3 \ --override "spec.template.spec.tolerations[0].key"="dedicated" \ --override "spec.template.spec.tolerations[0].operator"="Equal" \ --override "spec.template.spec.tolerations[0].value"="helm" \ - --override "spec.template.spec.tolerations[0].effect"="NoSchedule" \ - --tiller-namespace "${TILLER_NAMESPACE}" + --override "spec.template.spec.tolerations[0].effect"="NoSchedule" - kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy" + kubectl rollout status --namespace "${namespace}" --watch "deployment/tiller-deploy" - if ! helm version --debug --tiller-namespace "${TILLER_NAMESPACE}"; then + if ! helm version --tiller-namespace "${namespace}" --debug; then echo "Failed to init Tiller." return 1 fi } function install_external_dns() { - local release_name="dns-gitlab-review-app" + local namespace="${KUBE_NAMESPACE}" + local release="dns-gitlab-review-app" local domain domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}') echoinfo "Installing external DNS for domain ${domain}..." true - if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${KUBE_NAMESPACE}" "${release_name}" ; then + if ! deploy_exists "${namespace}" "${release}" || previous_deploy_failed "${namespace}" "${release}" ; then echoinfo "Installing external-dns Helm chart" - helm repo update + helm repo update --tiller-namespace "${namespace}" + # Default requested: CPU => 0, memory => 0 - helm install stable/external-dns --version '^2.2.1' \ - -n "${release_name}" \ - --namespace "${KUBE_NAMESPACE}" \ + # Chart > 2.6.1 has a problem with AWS so we're pinning it for now. + # See https://gitlab.com/gitlab-org/gitlab/issues/37269 and https://github.com/kubernetes-sigs/external-dns/issues/1262 + helm install stable/external-dns \ + --tiller-namespace "${namespace}" \ + --namespace "${namespace}" \ + --version '2.6.1' \ + --name "${release}" \ --set provider="aws" \ --set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \ --set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \ --set aws.zoneType="public" \ --set aws.batchChangeSize=400 \ --set domainFilters[0]="${domain}" \ - --set txtOwnerId="${KUBE_NAMESPACE}" \ + --set txtOwnerId="${namespace}" \ --set rbac.create="true" \ --set policy="sync" \ --set resources.requests.cpu=50m \ @@ -177,23 +215,35 @@ function install_external_dns() { } function create_application_secret() { - echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true - - kubectl create secret generic -n "$KUBE_NAMESPACE" \ - "${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \ - --from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \ - --dry-run -o json | kubectl apply -f - + local namespace="${KUBE_NAMESPACE}" + local release="${CI_ENVIRONMENT_SLUG}" + local initial_root_password_shared_secret + local gitlab_license_shared_secret + + initial_root_password_shared_secret=$(kubectl get secret --namespace ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name shared-gitlab-initial-root-password | tail -n 1) + if [[ "${initial_root_password_shared_secret}" == "" ]]; then + echoinfo "Creating the 'shared-gitlab-initial-root-password' secret in the ${namespace} namespace..." true + kubectl create secret generic --namespace "${namespace}" \ + "shared-gitlab-initial-root-password" \ + --from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \ + --dry-run -o json | kubectl apply -f - + else + echoinfo "The 'shared-gitlab-initial-root-password' secret already exists in the ${namespace} namespace." + fi if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi - echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-license secret in the ${KUBE_NAMESPACE} namespace..." true - - echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab - - kubectl create secret generic -n "$KUBE_NAMESPACE" \ - "${CI_ENVIRONMENT_SLUG}-gitlab-license" \ - --from-file=license=/tmp/license.gitlab \ - --dry-run -o json | kubectl apply -f - + gitlab_license_shared_secret=$(kubectl get secret --namespace ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name shared-gitlab-license | tail -n 1) + if [[ "${gitlab_license_shared_secret}" == "" ]]; then + echoinfo "Creating the 'shared-gitlab-license' secret in the ${namespace} namespace..." true + echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab + kubectl create secret generic --namespace "${namespace}" \ + "shared-gitlab-license" \ + --from-file=license=/tmp/license.gitlab \ + --dry-run -o json | kubectl apply -f - + else + echoinfo "The 'shared-gitlab-license' secret already exists in the ${namespace} namespace." + fi } function download_chart() { @@ -217,13 +267,14 @@ function base_config_changed() { } function deploy() { - local name="$CI_ENVIRONMENT_SLUG" + local namespace="${KUBE_NAMESPACE}" + local release="${CI_ENVIRONMENT_SLUG}" local edition="${GITLAB_EDITION-ce}" local base_config_file_ref="master" - if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="$CI_COMMIT_SHA"; fi + if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml" - echoinfo "Deploying ${name}..." true + echoinfo "Deploying ${release}..." true IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror" gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${edition}" @@ -237,47 +288,49 @@ function deploy() { create_application_secret HELM_CMD=$(cat << EOF - helm upgrade --install \ + helm upgrade \ + --tiller-namespace="${namespace}" \ + --namespace="${namespace}" \ + --install \ --wait \ --timeout 900 \ - --set ci.branch="$CI_COMMIT_REF_NAME" \ - --set ci.commit.sha="$CI_COMMIT_SHORT_SHA" \ - --set ci.job.url="$CI_JOB_URL" \ - --set ci.pipeline.url="$CI_PIPELINE_URL" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set global.hosts.hostSuffix="$HOST_SUFFIX" \ - --set global.hosts.domain="$REVIEW_APPS_DOMAIN" \ - --set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \ - --set gitlab.migrations.image.tag="$CI_COMMIT_REF_SLUG" \ - --set gitlab.gitaly.image.repository="$gitlab_gitaly_image_repository" \ - --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \ - --set gitlab.gitlab-shell.image.repository="$gitlab_shell_image_repository" \ - --set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \ - --set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \ - --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_SLUG" \ - --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \ - --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_SLUG" \ - --set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \ - --set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_SLUG" \ - --set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \ - --set gitlab.task-runner.image.tag="$CI_COMMIT_REF_SLUG" + --set ci.branch="${CI_COMMIT_REF_NAME}" \ + --set ci.commit.sha="${CI_COMMIT_SHORT_SHA}" \ + --set ci.job.url="${CI_JOB_URL}" \ + --set ci.pipeline.url="${CI_PIPELINE_URL}" \ + --set releaseOverride="${release}" \ + --set global.hosts.hostSuffix="${HOST_SUFFIX}" \ + --set global.hosts.domain="${REVIEW_APPS_DOMAIN}" \ + --set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \ + --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \ + --set gitlab.gitaly.image.tag="v${GITALY_VERSION}" \ + --set gitlab.gitlab-shell.image.repository="${gitlab_shell_image_repository}" \ + --set gitlab.gitlab-shell.image.tag="v${GITLAB_SHELL_VERSION}" \ + --set gitlab.sidekiq.image.repository="${gitlab_sidekiq_image_repository}" \ + --set gitlab.sidekiq.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.unicorn.image.repository="${gitlab_unicorn_image_repository}" \ + --set gitlab.unicorn.image.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.unicorn.workhorse.image="${gitlab_workhorse_image_repository}" \ + --set gitlab.unicorn.workhorse.tag="${CI_COMMIT_REF_SLUG}" \ + --set gitlab.task-runner.image.repository="${gitlab_task_runner_image_repository}" \ + --set gitlab.task-runner.image.tag="${CI_COMMIT_REF_SLUG}" EOF ) if [ -n "${REVIEW_APPS_EE_LICENSE}" ]; then HELM_CMD=$(cat << EOF ${HELM_CMD} \ - --set global.gitlab.license.secret="${CI_ENVIRONMENT_SLUG}-gitlab-license" + --set global.gitlab.license.secret="shared-gitlab-license" EOF ) fi HELM_CMD=$(cat << EOF ${HELM_CMD} \ - --namespace="$KUBE_NAMESPACE" \ --version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \ -f "${base_config_file}" \ - "${name}" . + "${release}" . EOF ) @@ -288,11 +341,14 @@ EOF } function display_deployment_debug() { + local namespace="${KUBE_NAMESPACE}" + local release="${CI_ENVIRONMENT_SLUG}" + # Get all pods for this release - echoinfo "Pods for release ${CI_ENVIRONMENT_SLUG}" - kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} + echoinfo "Pods for release ${release}" + kubectl get pods --namespace "${namespace}" -lrelease=${release} # Get all non-completed jobs - echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}" - kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1 + echoinfo "Unsuccessful Jobs for release ${release}" + kubectl get jobs --namespace "${namespace}" -lrelease=${release} --field-selector=status.successful!=1 } diff --git a/scripts/security-harness b/scripts/security-harness index 8369cf06223..a1642489fe2 100755 --- a/scripts/security-harness +++ b/scripts/security-harness @@ -1,57 +1,99 @@ #!/usr/bin/env ruby +# frozen_string_literal: true + require 'digest' require 'fileutils' -harness_path = File.expand_path('../.git/security_harness', __dir__) -hook_path = File.expand_path("../.git/hooks/pre-push", __dir__) +if ENV['NO_COLOR'] + SHELL_RED = '' + SHELL_GREEN = '' + SHELL_YELLOW = '' + SHELL_CLEAR = '' +else + SHELL_RED = "\e[1;31m" + SHELL_GREEN = "\e[1;32m" + SHELL_YELLOW = "\e[1;33m" + SHELL_CLEAR = "\e[0m" +end -if File.exist?(hook_path) - # Deal with a pre-existing hook - source_sum = Digest::SHA256.hexdigest(DATA.read) - dest_sum = Digest::SHA256.file(hook_path).hexdigest +HOOK_PATH = File.expand_path("../.git/hooks/pre-push", __dir__) +HOOK_DATA = <<~HOOK + #!/bin/bash - if source_sum != dest_sum - puts "#{hook_path} exists and is different from our hook!" - puts "Remove it and re-run this script to continue." + set -e - exit 1 - end -else - File.open(hook_path, 'w') do |file| - IO.copy_stream(DATA, file) - end + url="$2" + harness=`dirname "$0"`/../security_harness + + if [ -e "$harness" ] + then + if [[ ("$url" != *"dev.gitlab.org"*) && ("$url" != *"gitlab-org/security/"*) ]] + then + echo "Pushing to remotes other than dev.gitlab.org and gitlab.com/gitlab-org/security has been disabled!" + echo "Run scripts/security-harness to disable this check." + echo + + exit 1 + fi + fi +HOOK - File.chmod(0755, hook_path) +def write_hook + FileUtils.mkdir_p(File.dirname(HOOK_PATH)) + File.open(HOOK_PATH, 'w') do |file| + file.write(HOOK_DATA) + end + File.chmod(0755, HOOK_PATH) end # Toggle the harness on or off -if File.exist?(harness_path) - FileUtils.rm(harness_path) +def toggle + harness_path = File.expand_path('../.git/security_harness', __dir__) - puts "Security harness removed -- you can now push to all remotes." -else - FileUtils.touch(harness_path) + if File.exist?(harness_path) + FileUtils.rm(harness_path) - puts "Security harness installed -- you will only be able to push to dev.gitlab.org!" -end + puts "#{SHELL_YELLOW}Security harness removed -- you can now push to all remotes.#{SHELL_CLEAR}" + else + FileUtils.touch(harness_path) -__END__ -#!/bin/bash + puts "#{SHELL_GREEN}Security harness installed -- you will only be able to push to dev.gitlab.org or gitlab.com/gitlab-org/security!#{SHELL_CLEAR}" + end +end -set -e +# If we were to change the script and then check for a pre-existing hook before +# writing, the check would fail even if the user had an unmodified version of +# the old hook. Checking previous version hashes allows us to safely overwrite a +# script that differs from the current version, as long as it's an old one and +# not custom. +def previous_version?(dest_sum) + # SHA256 hashes of previous iterations of the script contained in `DATA` + %w[ + 010bf0363a911ebab2bd5728d80795ed02388da51815f0b2530d08ae8ac574f0 + ].include?(dest_sum) +end -url="$2" -harness=`dirname "$0"`/../security_harness +if !File.exist?(HOOK_PATH) + write_hook + toggle +else + # Deal with a pre-existing hook + source_sum = Digest::SHA256.hexdigest(HOOK_DATA) + dest_sum = Digest::SHA256.file(HOOK_PATH).hexdigest -if [ -e "$harness" ] -then - if [[ "$url" != *"dev.gitlab.org"* ]] - then - echo "Pushing to remotes other than dev.gitlab.org has been disabled!" - echo "Run scripts/security-harness to disable this check." - echo + if previous_version?(dest_sum) + # Upgrading from a previous version, update in-place + write_hook + toggle + elsif source_sum != dest_sum + # Pre-existing hook we didn't create; do nothing + puts "#{SHELL_RED}#{HOOK_PATH} exists and is different from our hook!" + puts "Remove it and re-run this script to continue.#{SHELL_CLEAR}" exit 1 - fi -fi + else + # No hook update needed, just toggle + toggle + end +end diff --git a/scripts/static-analysis b/scripts/static-analysis index b7f7100c365..1392a4f6a23 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -35,7 +35,8 @@ def jobs_to_run(node_index, node_total) %w[yarn run prettier-all], %w[bundle exec rubocop --parallel], %w[scripts/lint-conflicts.sh], - %w[scripts/lint-rugged] + %w[scripts/lint-rugged], + %w[scripts/frontend/check_no_partial_karma_jest.sh] ] case node_total diff --git a/scripts/sync-stable-branch.sh b/scripts/sync-stable-branch.sh index fc62453d743..b44bf26a151 100644 --- a/scripts/sync-stable-branch.sh +++ b/scripts/sync-stable-branch.sh @@ -23,10 +23,24 @@ then exit 1 fi +if [[ "$SOURCE_PROJECT" == '' ]] +then + echo 'The variable SOURCE_PROJECT must be set to a non-empy value' + exit 1 +fi + +if [[ "$TARGET_PROJECT" == '' ]] +then + echo 'The variable TARGET_PROJECT must be set to a non-empy value' + exit 1 +fi + curl -X POST \ -F token="$MERGE_TRAIN_TRIGGER_TOKEN" \ -F ref=master \ -F "variables[MERGE_FOSS]=1" \ -F "variables[SOURCE_BRANCH]=$CI_COMMIT_REF_NAME" \ -F "variables[TARGET_BRANCH]=${CI_COMMIT_REF_NAME/-ee/}" \ + -F "variables[SOURCE_PROJECT]=$SOURCE_PROJECT" \ + -F "variables[TARGET_PROJECT]=$TARGET_PROJECT" \ "$MERGE_TRAIN_TRIGGER_URL" diff --git a/scripts/trigger-build b/scripts/trigger-build index 74c1df258c0..537b2692b27 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -71,7 +71,7 @@ module Trigger # Can be overridden def version_param_value(version_file) - File.read(version_file).strip + ENV[version_file]&.strip || File.read(version_file).strip end def variables diff --git a/scripts/utils.sh b/scripts/utils.sh index f0f08e2e1c5..7eae9531f74 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -14,11 +14,7 @@ function retry() { } function setup_db_user_only() { - if [ "$GITLAB_DATABASE" = "postgresql" ]; then - source scripts/create_postgres_user.sh - else - source scripts/create_mysql_user.sh - fi + source scripts/create_postgres_user.sh } function setup_db() { @@ -26,10 +22,6 @@ function setup_db() { bundle exec rake db:drop db:create db:schema:load db:migrate - if [ "$GITLAB_DATABASE" = "mysql" ]; then - bundle exec rake add_limits_mysql - fi - bundle exec rake gitlab:db:setup_ee } @@ -42,7 +34,8 @@ function install_api_client_dependencies_with_apt() { } function install_gitlab_gem() { - gem install gitlab --no-document + gem install httparty --no-document --version 0.17.3 + gem install gitlab --no-document --version 4.13.0 } function echoerr() { |