From 95a5390f185351f20081097da18c6f364ce96f8a Mon Sep 17 00:00:00 2001 From: Evan Read Date: Thu, 24 Nov 2022 16:15:29 +0000 Subject: Add ShellCheck to project --- scripts/check-lunr-index.sh | 8 +++-- scripts/compress_images.sh | 9 +++--- scripts/deploy-review-app | 68 ------------------------------------------ scripts/deploy-review-app.sh | 68 ++++++++++++++++++++++++++++++++++++++++++ scripts/minify-assets.sh | 16 +++++----- scripts/normalize-links.sh | 48 +++++++++++------------------ scripts/review-replace-urls | 4 --- scripts/review-replace-urls.sh | 4 +++ 8 files changed, 106 insertions(+), 119 deletions(-) delete mode 100755 scripts/deploy-review-app create mode 100755 scripts/deploy-review-app.sh delete mode 100755 scripts/review-replace-urls create mode 100755 scripts/review-replace-urls.sh (limited to 'scripts') diff --git a/scripts/check-lunr-index.sh b/scripts/check-lunr-index.sh index b3019e2d..2e368a38 100755 --- a/scripts/check-lunr-index.sh +++ b/scripts/check-lunr-index.sh @@ -18,17 +18,18 @@ COLOR_RESET="\e[39m" if [ "$CI" = "true" ]; then div_check=$(docker run --rm "$IMAGE_NAME" grep -o js-lunr-form "/usr/share/nginx/html/$GITLAB_VERSION/index.html") - index_check=$(docker run --rm "$IMAGE_NAME" ls "/usr/share/nginx/html/$GITLAB_VERSION/assets/javascripts/lunr-index.json" | wc -l) + index_check=$(docker run --rm "$IMAGE_NAME" find "/usr/share/nginx/html/$GITLAB_VERSION/assets/javascripts/lunr-index.json" | wc -l) else div_check=$(grep -o js-lunr-form public/index.html) - index_check=$(ls public/assets/javascripts/lunr-index.json | wc -l) + index_check=$(find public/assets/javascripts/lunr-index.json | wc -l) fi -if [ $index_check != 1 ]; +if [ "$index_check" != 1 ]; then # shellcheck disable=2059 printf "${COLOR_RED}ERROR: lunr.js index is not built!\n" printf " Did you forget to run 'make build-lunr-index'?\n" + # shellcheck disable=2059 printf " For more information, see https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/docsearch.md#lunrjs-search${COLOR_RESET}\n" exit 1; else @@ -37,6 +38,7 @@ else # shellcheck disable=2059 printf "${COLOR_RED}ERROR: lunr.js index is found, but not enabled!\n" printf " Did you forget to build the site with ALGOLIA_SEARCH='false'?\n" + # shellcheck disable=2059 printf " For more information, see https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/docsearch.md#lunrjs-search${COLOR_RESET}\n" exit 1; else diff --git a/scripts/compress_images.sh b/scripts/compress_images.sh index 87993268..dd839d79 100755 --- a/scripts/compress_images.sh +++ b/scripts/compress_images.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash TARGET="$1" VER="$2" @@ -23,7 +23,8 @@ if ! [ -d "$TARGET/$VER" ]; then fi # Compress images -for image in $(find ${TARGET}/${VER}/ -name "*.png") - do $PNG $image - echo "Compressing $image" +# shellcheck disable=SC2044 +for image in $(find "${TARGET}/${VER}/" -name "*.png") + do echo "Compressing $image" + $PNG "$image" done diff --git a/scripts/deploy-review-app b/scripts/deploy-review-app deleted file mode 100755 index fa2bc0dc..00000000 --- a/scripts/deploy-review-app +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# -# Copied and adapted from https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/10b95368133ea0a23326d293b7a80fc71317d011/scripts/deploy -# - -# Exponential backoff, found from StackOverflow at https://stackoverflow.com/questions/8350942/how-to-re-run-the-curl-command-automatically-when-the-error-occurs/8351489#8351489 -# Retries a command a with backoff. -# -# The retry count is given by ATTEMPTS (default 5), the -# initial backoff timeout is given by TIMEOUT in seconds -# (default 1.) -# -# Successive backoffs double the timeout. -# -# Beware of set -e killing your whole script! -function with_backoff { - local max_attempts=${ATTEMPTS-5} - local timeout=${TIMEOUT-1} - local attempt=0 - local exitCode=0 - - while [[ $attempt < $max_attempts ]] - do - "$@" - exitCode=$? - - if [[ $exitCode == 0 ]] - then - break - fi - - echo "Failure! Retrying in $timeout.." 1>&2 - sleep $timeout - attempt=$(( attempt + 1 )) - timeout=$(( timeout * 2 )) - done - - if [[ $exitCode != 0 ]] - then - echo "You've failed me for the last time! ($@)" 1>&2 - fi - - return $exitCode -} - -echo "Starting deploy for review app." - -gcp_project=$GCP_PROJECT_REVIEW_APPS -gcp_bucket=$GCP_BUCKET_REVIEW_APPS -gcp_service_account_key=$GCP_SERVICE_ACCOUNT_KEY_REVIEW_APPS -cache_control_max_age='60' -src='public/' -dest="gs://$gcp_bucket/$CI_COMMIT_REF_SLUG$REVIEW_SLUG" - -echo "$gcp_service_account_key" > key.json -gcloud auth activate-service-account --key-file key.json -gcloud config set project "$gcp_project" - -if [ "$DEPLOY_DELETE_APP" = 'true' ]; then - echo "Deleting review app from ${dest}..." - echo "gsutil -m rm -r \"$dest\"" - gsutil -m rm -r "$dest" -else - echo "Deploying review app to ${dest}..." - echo "gsutil -h \"Cache-Control:public, max-age=$cache_control_max_age\" -m rsync -j css,html,js,txt -r -d \"$src\" \"$dest\"" - with_backoff gsutil -h "Cache-Control:public, max-age=$cache_control_max_age" -m rsync -j css,html,js,txt -r -d "$src" "$dest" -fi diff --git a/scripts/deploy-review-app.sh b/scripts/deploy-review-app.sh new file mode 100755 index 00000000..ea716099 --- /dev/null +++ b/scripts/deploy-review-app.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# +# Copied and adapted from https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/10b95368133ea0a23326d293b7a80fc71317d011/scripts/deploy +# + +# Exponential backoff, found from StackOverflow at https://stackoverflow.com/questions/8350942/how-to-re-run-the-curl-command-automatically-when-the-error-occurs/8351489#8351489 +# Retries a command a with backoff. +# +# The retry count is given by ATTEMPTS (default 5), the +# initial backoff timeout is given by TIMEOUT in seconds +# (default 1.) +# +# Successive backoffs double the timeout. +# +# Beware of set -e killing your whole script! +function with_backoff { + local max_attempts=${ATTEMPTS-5} + local timeout=${TIMEOUT-1} + local attempt=0 + local exitCode=0 + + while [[ $attempt < $max_attempts ]] + do + "$@" + exitCode=$? + + if [[ $exitCode == 0 ]] + then + break + fi + + echo "Failure! Retrying in $timeout.." 1>&2 + sleep "$timeout" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done + + if [[ $exitCode != 0 ]] + then + echo "You've failed me for the last time! ($*)" 1>&2 + fi + + return $exitCode +} + +echo "Starting deploy for review app." + +gcp_project=$GCP_PROJECT_REVIEW_APPS +gcp_bucket=$GCP_BUCKET_REVIEW_APPS +gcp_service_account_key=$GCP_SERVICE_ACCOUNT_KEY_REVIEW_APPS +cache_control_max_age='60' +src='public/' +dest="gs://$gcp_bucket/$CI_COMMIT_REF_SLUG$REVIEW_SLUG" + +echo "$gcp_service_account_key" > key.json +gcloud auth activate-service-account --key-file key.json +gcloud config set project "$gcp_project" + +if [ "$DEPLOY_DELETE_APP" = 'true' ]; then + echo "Deleting review app from ${dest}..." + echo "gsutil -m rm -r \"$dest\"" + gsutil -m rm -r "$dest" +else + echo "Deploying review app to ${dest}..." + echo "gsutil -h \"Cache-Control:public, max-age=$cache_control_max_age\" -m rsync -j css,html,js,txt -r -d \"$src\" \"$dest\"" + with_backoff gsutil -h "Cache-Control:public, max-age=$cache_control_max_age" -m rsync -j css,html,js,txt -r -d "$src" "$dest" +fi diff --git a/scripts/minify-assets.sh b/scripts/minify-assets.sh index 80bc7b91..cde1c20d 100755 --- a/scripts/minify-assets.sh +++ b/scripts/minify-assets.sh @@ -1,10 +1,10 @@ -#!/bin/sh +#!/usr/bin/env bash TARGET="$1" VER="$2" MINIFY_FLAGS="--html-keep-document-tags --html-keep-whitespace --recursive" -if [ -z "$TARGET" -o -z "$VER" ]; then +if [ -z "$TARGET" ] || [ -z "$VER" ]; then echo "Usage: $0 " echo "Either or is missing. Exiting." exit 1 @@ -16,9 +16,7 @@ if ! [ -d "$TARGET" ]; then fi # Check if minify is in the PATH -which minify > /dev/null 2>&1 -# Check if the previous command has a 0 exit status -if [ $? -eq 0 ] +if which minify > /dev/null 2>&1 then MINIFY_BIN=$(which minify) else @@ -35,9 +33,9 @@ fi # Minify assets printf "Optimizing assets..." -printf "HTML..."; $MINIFY_BIN $MINIFY_FLAGS --type=html --match="\.html$" -o ${TARGET}/${VER}/ ${TARGET}/${VER} || true -printf "CSS..." ; $MINIFY_BIN $MINIFY_FLAGS --type=css --match="\.css$" -o ${TARGET}/${VER}/ ${TARGET}/${VER} || true -printf "JSON..."; $MINIFY_BIN $MINIFY_FLAGS --type=json --match="\.json$" -o ${TARGET}/${VER}/ ${TARGET}/${VER} || true -printf "SVG..." ; $MINIFY_BIN $MINIFY_FLAGS --type=svg --match="\.svg$" -o ${TARGET}/${VER}/ ${TARGET}/${VER} || true +printf "HTML..."; $MINIFY_BIN "$MINIFY_FLAGS" --type=html --match="\.html$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}" || true +printf "CSS..." ; $MINIFY_BIN "$MINIFY_FLAGS" --type=css --match="\.css$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}" || true +printf "JSON..."; $MINIFY_BIN "$MINIFY_FLAGS" --type=json --match="\.json$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}" || true +printf "SVG..." ; $MINIFY_BIN "$MINIFY_FLAGS" --type=svg --match="\.svg$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}" || true echo "Done" diff --git a/scripts/normalize-links.sh b/scripts/normalize-links.sh index 3f5d8e46..fd534f83 100755 --- a/scripts/normalize-links.sh +++ b/scripts/normalize-links.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash TARGET="$1" # The directory that has all the HTML files including versions. # Usually public/ locally and /site in the Docker image. @@ -34,70 +34,56 @@ fi ## Relative URLs ## echo "Replace relative URLs in $TARGET/$VER for /ee/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/ee/#="/'"$VER"'/ee/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/ee/#="/'"$VER"'/ee/#g' echo "Replace relative URLs in $TARGET/$VER for /runner/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/runner/#="/'"$VER"'/runner/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/runner/#="/'"$VER"'/runner/#g' echo "Replace relative URLs in $TARGET/$VER for /omnibus/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/omnibus/#="/'"$VER"'/omnibus/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/omnibus/#="/'"$VER"'/omnibus/#g' echo "Replace relative URLs in $TARGET/$VER for /charts/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/charts/#="/'"$VER"'/charts/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/charts/#="/'"$VER"'/charts/#g' echo "Replace relative URLs in $TARGET/$VER for /operator/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/operator/#="/'"$VER"'/operator/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/operator/#="/'"$VER"'/operator/#g' echo "Replace relative URLs in $TARGET/$VER for /assets/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/assets/#="/'"$VER"'/assets/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/assets/#="/'"$VER"'/assets/#g' echo "Replace relative URLs in $TARGET/$VER for /frontend/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/frontend/#="/'"$VER"'/frontend/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/frontend/#="/'"$VER"'/frontend/#g' echo "Replace relative URLs in $TARGET/$VER for /" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's###g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's###g' echo "Replace relative URLs in $TARGET/$VER for opensearch.xml" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/opensearch.xml#="/'"$VER"'/opensearch.xml#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="/opensearch.xml#="/'"$VER"'/opensearch.xml#g' ## ## Full URLs ## echo "Replace full URLs in $TARGET/$VER for /ee/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/ee/#="/'"$VER"'/ee/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/ee/#="/'"$VER"'/ee/#g' echo "Replace full URLs in $TARGET/$VER for /runner/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/runner/#="/'"$VER"'/runner/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/runner/#="/'"$VER"'/runner/#g' echo "Replace full URLs in $TARGET/$VER for /omnibus/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/omnibus/#="/'"$VER"'/omnibus/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/omnibus/#="/'"$VER"'/omnibus/#g' echo "Replace full URLs in $TARGET/$VER for /charts/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/charts/#="/'"$VER"'/charts/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/charts/#="/'"$VER"'/charts/#g' echo "Replace full URLs in $TARGET/$VER for /operator/" -find ${TARGET}/$VER -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/operator/#="/'"$VER"'/operator/#g' +find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 sed -i 's#="https://docs.gitlab.com/operator/#="/'"$VER"'/operator/#g' echo "Fix URLs inside the sitemap" -find ${TARGET}/$VER -type f -name 'sitemap.xml' -print0 | xargs -0 sed -i 's#docs.gitlab.com/#docs.gitlab.com/'"$VER"'/#g' - -## -## In order to have clean URLs, we symlink README.html to index.html. -## That way, visiting https://docs.gitlab.com/ee/ would be the same as -## visiting https://docs.gitlab.com/ee/{README.html,index.html} -## For 13.9 and later, there's a raketask that is run instead of the -## command below. If the raketask is present, skip the command. -## -bundle exec rake -T | grep symlink_readmes -if [ $? -eq 1 ] -then - echo "Symlink all README.html to index.html" - for i in `find ${TARGET}/${VER} -name README.html`; do ln -sf README.html $(dirname $i)/index.html; done -fi +find "${TARGET}/$VER" -type f -name 'sitemap.xml' -print0 | xargs -0 sed -i 's#docs.gitlab.com/#docs.gitlab.com/'"$VER"'/#g' ## ## Don't deploy the CE docs since they are identical to the EE ones. ## https://gitlab.com/gitlab-org/gitlab-docs/issues/418 ## echo "Remove CE dir and symlink EE to CE" -if [ -d "${TARGET}/${VER}/ce/" ]; then cd ${TARGET}/${VER} && rm -r ce && ln -s ee ce; fi +if [ -d "${TARGET}/${VER}/ce/" ]; then cd "${TARGET}/${VER}" && rm -r ce && ln -s ee ce; fi diff --git a/scripts/review-replace-urls b/scripts/review-replace-urls deleted file mode 100755 index 108b763a..00000000 --- a/scripts/review-replace-urls +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -# Note this uses linux-specific (GNU) find syntax, it does not work with MacOS (BSD) find syntax because of the usage of the 'regextype' flag (and possibly other reasons) -find public/ -type f -regextype egrep -iregex ".*\.(html|js|css|json|xml|txt)" -exec sed --in-place "s#https\?://docs.gitlab.com#https://$CI_COMMIT_REF_SLUG$REVIEW_SLUG.docs.gitlab-review.app#g" "{}" +; diff --git a/scripts/review-replace-urls.sh b/scripts/review-replace-urls.sh new file mode 100755 index 00000000..108b763a --- /dev/null +++ b/scripts/review-replace-urls.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Note this uses linux-specific (GNU) find syntax, it does not work with MacOS (BSD) find syntax because of the usage of the 'regextype' flag (and possibly other reasons) +find public/ -type f -regextype egrep -iregex ".*\.(html|js|css|json|xml|txt)" -exec sed --in-place "s#https\?://docs.gitlab.com#https://$CI_COMMIT_REF_SLUG$REVIEW_SLUG.docs.gitlab-review.app#g" "{}" +; -- cgit v1.2.3