Redirect for ${VER}

From 5eeebbc5b2996b2e36c06700adf5677e53aa248d Mon Sep 17 00:00:00 2001 From: Sarah German Date: Fri, 5 May 2023 19:05:23 +0000 Subject: Merge branch 'axil-fix-minify-run' into 'main' Change logic of input and output targets of post-processing scripts Closes #1612 See merge request https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/3791 Merged-by: Sarah German Approved-by: Sarah German Reviewed-by: Achilleas Pipinellis Co-authored-by: Achilleas Pipinellis (cherry picked from commit 0a602766667fc93591571f37f63b9bc03d568fd3) 27107f1f Change logic of input and output targets of post-processing scripts 6bc5c46d Apply 2 suggestion(s) to 1 file(s) --- .gitlab/ci/build-and-deploy.gitlab-ci.yml | 12 ++--- dockerfiles/single.Dockerfile | 32 +++++++------ scripts/compress_images.sh | 20 ++------ scripts/minify-assets.sh | 77 ++++++++++++++++--------------- scripts/normalize-links.sh | 63 +++++++++++-------------- 5 files changed, 93 insertions(+), 111 deletions(-) diff --git a/.gitlab/ci/build-and-deploy.gitlab-ci.yml b/.gitlab/ci/build-and-deploy.gitlab-ci.yml index 7a79135d..42898480 100644 --- a/.gitlab/ci/build-and-deploy.gitlab-ci.yml +++ b/.gitlab/ci/build-and-deploy.gitlab-ci.yml @@ -32,15 +32,11 @@ - bundle exec rake redirects # Build the Lunr.js search index if needed - if [[ "$SEARCH_BACKEND" == "lunr" ]]; then node scripts/lunr/preindex.js; fi - # Calculate sizes before and after minifying/gzipping the static files (HTML, CSS, JS) - - SIZE_BEFORE_MINIFY=$(du -sh public/ | awk '{print $1}') # Minify the assets of the resulting site - - cd public - - ../scripts/minify-assets.sh ./ ./ - - cd .. - - SIZE_AFTER_MINIFY=$(du -sh public/ | awk '{print $1}') - # Print size results - - echo -e "Size before minifying ......... $SIZE_BEFORE_MINIFY\nSize after minifying ................... $SIZE_AFTER_MINIFY" + - mkdir dest/ + - scripts/minify-assets.sh dest/ public/ + - rm -rf public/ + - mv dest/public public/ # # The script lines for gzipping the site (not used in upstream review apps, to speed up pipelines) diff --git a/dockerfiles/single.Dockerfile b/dockerfiles/single.Dockerfile index 5671da42..5c864d3c 100644 --- a/dockerfiles/single.Dockerfile +++ b/dockerfiles/single.Dockerfile @@ -63,19 +63,21 @@ RUN yarn install --frozen-lockfile \ RUN if [ "$SEARCH_BACKEND" = "lunr" ]; then make build-lunr-index; fi -# Move generated HTML to /site -RUN mkdir /site \ - && mv public "/site/${VER}" - -# Do some HTML post-processing on the archive, compress images, and minify assets -RUN /source/scripts/normalize-links.sh /site "${VER}" \ - && /source/scripts/compress_images.sh /site "${VER}" \ - # ATTENTION: This should be the last script to run - && /source/scripts/minify-assets.sh /site "${VER}" +# Run post-processing on archive: +# +# 1. Normalize the links in /source/public using version $VER. +# 2. Compress images in /source/public. +# 3. Minify the files in /source/public into /dest, creating /dest/public. Must run last. +# 4. Rename /dest/public to /dest/$VER +RUN /source/scripts/normalize-links.sh /source/public $VER \ + && /source/scripts/compress_images.sh /source/public \ + && mkdir /dest \ + && /source/scripts/minify-assets.sh /dest /source/public \ + && mv /dest/public "/dest/${VER}" # Make an index.html and 404.html which will redirect / to /${VER}/ -RUN echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /site/index.html \ - && echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /site/404.html +RUN echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /dest/index.html \ + && echo "Redirect for ${VER}

If you are not redirected automatically, click here.

" > /dest/404.html #- End of builder build stage -# @@ -87,14 +89,14 @@ RUN echo "Redirect for ${VER} " +if [ -z "$INPUT" ]; then + echo "Usage: $0 " echo "No target provided. Exiting." exit 1 fi -if [ -z "$VER" ]; then - echo "Usage: $0 " - echo "No version provided. Exiting." - exit 1 -fi - -if ! [ -d "$TARGET/$VER" ]; then - echo "Target directory $TARGET/$VER does not exist. Exiting." - exit 1 -fi - # Compress images # shellcheck disable=SC2044 -for image in $(find "${TARGET}/${VER}/" -name "*.png") +for image in $(find "${INPUT}/" -name "*.png") do echo "Compressing $image" $PNG "$image" done diff --git a/scripts/minify-assets.sh b/scripts/minify-assets.sh index aa27512e..edcb3288 100755 --- a/scripts/minify-assets.sh +++ b/scripts/minify-assets.sh @@ -1,22 +1,22 @@ #!/usr/bin/env bash # shellcheck disable=SC2059 -TARGET="$1" -VER="$2" -MINIFY_FLAGS=("--html-keep-document-tags" "--html-keep-whitespace" "--recursive") +DEST="$1" +SOURCE="$2" +MINIFY_FLAGS=("--html-keep-document-tags" "--html-keep-whitespace" "--recursive" "--sync") COLOR_RED="\e[31m" COLOR_GREEN="\e[32m" COLOR_RESET="\e[39m" -if [ -z "$TARGET" ] || [ -z "$VER" ]; then +if [ -z "$DEST" ] || [ -z "$SOURCE" ]; then echo "Usage: $0 " - printf "${COLOR_RED}ERROR: Either or is missing.${COLOR_RESET}\n" + printf "${COLOR_RED}ERROR: Either or is missing.${COLOR_RESET}\n" exit 1 fi -if ! [ -d "$TARGET" ]; then - printf "${COLOR_RED}ERROR: Target directory $TARGET does not exist.${COLOR_RESET}\n" +if ! [ -d "$DEST" ]; then + printf "${COLOR_RED}ERROR: Output directory $DEST does not exist.${COLOR_RESET}\n" exit 1 fi @@ -30,41 +30,46 @@ else then MINIFY_BIN=/scripts/minify else - printf "${COLOR_RED}ERROR: minify not found in PATH. Run 'make setup'.${COLOR_RESET}\n" + printf "${COLOR_RED}ERROR: minify not found in PATH. Run 'make setup'.${COLOR_RESET}\n" exit 1 fi fi # Minify assets -printf "${COLOR_GREEN}INFO: Minifying HTML...${COLOR_RESET}\n" -if $MINIFY_BIN "${MINIFY_FLAGS[@]}" --type=html --match="\.html$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}"; then - printf "${COLOR_GREEN}INFO: HTML minified!${COLOR_RESET}\n" -else - printf "${COLOR_RED}ERROR: Couldn't minify HTML${COLOR_RESET}\n" - exit 1 -fi - -printf "${COLOR_GREEN}INFO: Minifying CSS...${COLOR_RESET}\n" -if $MINIFY_BIN "${MINIFY_FLAGS[@]}" --type=css --match="\.css$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}"; then - printf "${COLOR_GREEN}INFO: CSS minified!${COLOR_RESET}\n" -else - printf "${COLOR_RED}ERROR: Couldn't minify CSS${COLOR_RESET}\n" - exit 1 -fi - - -printf "${COLOR_GREEN}INFO: Minifying JSON...${COLOR_RESET}\n" -if $MINIFY_BIN "${MINIFY_FLAGS[@]}" --type=json --match="\.json$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}"; then - printf "${COLOR_GREEN}INFO: JSON minified!${COLOR_RESET}\n" -else - printf "${COLOR_RED}ERROR: Couldn't minify JSON${COLOR_RESET}\n" - exit 1 -fi +printf "${COLOR_GREEN}INFO: Minifying all assets...${COLOR_RESET}\n" +if $MINIFY_BIN "${MINIFY_FLAGS[@]}" --match="\.html$|\.css$|\.json|\.svg$" -o "${DEST}" "${SOURCE}"; then + printf "${COLOR_GREEN}INFO: Assets minified!${COLOR_RESET}\n" + # Calculate sizes before and after minifying/gzipping the static files (HTML, CSS, JS) + SIZE_BEFORE_MINIFY=$(du -sh "$SOURCE" | awk '{print $1}') + SIZE_AFTER_MINIFY=$(du -sh "$DEST" | awk '{print $1}') + # Print size results + printf "${COLOR_GREEN}INFO: Size before minifying: $SIZE_BEFORE_MINIFY ${COLOR_RESET}\n" + printf "${COLOR_GREEN}INFO: Size after minifying: $SIZE_AFTER_MINIFY ${COLOR_RESET}\n" -printf "${COLOR_GREEN}INFO: Minifying SVGs...${COLOR_RESET}\n" -if $MINIFY_BIN "${MINIFY_FLAGS[@]}" --type=svg --match="\.svg$" -o "${TARGET}/${VER}/" "${TARGET}/${VER}"; then - printf "${COLOR_GREEN}INFO: SVGs minified!${COLOR_RESET}\n" + # + # Test that minification worked by comparing the number of the source and + # destination files. + # + SOURCE_FILES_COUNT=$(find "$SOURCE" -type f | wc -l) + DEST_FILES_COUNT=$(find "$DEST" -type f | wc -l) + SOURCE_DIR_COUNT=$(find "$SOURCE" -type d | wc -l) + # Destination is always one directory deeper (more) than source, so we subtract one + DEST_DIR_COUNT=$(($(find "$DEST" -type d | wc -l) - 1)) + printf "${COLOR_GREEN}INFO: Checking that the number of source and destination files is the same...${COLOR_RESET}\n" + if [[ $SOURCE_FILES_COUNT -eq $DEST_FILES_COUNT ]]; then + printf "${COLOR_GREEN}SUCCESS: File count in $SOURCE and $DEST: $SOURCE_FILES_COUNT/$DEST_FILES_COUNT ${COLOR_RESET}\n" + else + printf "${COLOR_RED}ERROR: File count in $SOURCE and $DEST do not match: $SOURCE_FILES_COUNT/$DEST_FILES_COUNT ${COLOR_RESET}" + exit 1 + fi + printf "${COLOR_GREEN}INFO: Checking that the number of source and destination directories is the same...${COLOR_RESET}\n" + if [[ $SOURCE_DIR_COUNT -eq $DEST_DIR_COUNT ]]; then + printf "${COLOR_GREEN}SUCCESS: Directory count in $SOURCE and $DEST: $SOURCE_DIR_COUNT/$DEST_DIR_COUNT ${COLOR_RESET}\n" + else + printf "${COLOR_RED}ERROR: Directory count in $SOURCE and $DEST do not match: $SOURCE_DIR_COUNT/$DEST_DIR_COUNT ${COLOR_RESET}" + exit 1 + fi else - printf "${COLOR_RED}ERROR: Couldn't minify SVGs${COLOR_RESET}\n" + printf "${COLOR_RED}ERROR: Couldn't minify assets${COLOR_RESET}\n" exit 1 fi diff --git a/scripts/normalize-links.sh b/scripts/normalize-links.sh index 442a5f43..fee57f40 100755 --- a/scripts/normalize-links.sh +++ b/scripts/normalize-links.sh @@ -4,11 +4,8 @@ COLOR_RED="\e[31m" COLOR_GREEN="\e[32m" COLOR_RESET="\e[39m" -TARGET="$1" # The directory that has all the HTML files including versions. - # Usually public/ locally and /site in the Docker image. - -VER="$2" # The docs version which is a directory holding all the respective - # versioned site, for example 13.0/ +INPUT="$1" # The directory that has all the HTML files. +VER="$2" # The docs version to replace the URLs with. ## Check which OS the script runs from since sed behaves differently ## on macOS and Linux. For macOS, check if you're using the built-in sed. @@ -27,8 +24,8 @@ else SED="sed" fi -if [ -z "$TARGET" ]; then - echo "Usage: $0 " +if [ -z "$INPUT" ]; then + echo "Usage: $0 " echo "Example: $0 public 13.0" # shellcheck disable=2059 printf "${COLOR_RED}ERROR: No target provided.${COLOR_RESET}\n" @@ -36,19 +33,13 @@ if [ -z "$TARGET" ]; then fi if [ -z "$VER" ]; then - echo "Usage: $0 " + echo "Usage: $0 " echo "Example: $0 public 13.0" # shellcheck disable=2059 printf "${COLOR_RED}ERROR: No version provided.${COLOR_RESET}\n" exit 1 fi -if ! [ -d "$TARGET/$VER" ]; then - # shellcheck disable=2059 - printf "${COLOR_RED}ERROR: Target directory $TARGET/$VER does not exist.${COLOR_RESET}\n" - exit 1 -fi - ## ## In order for the version to be correct, we need to replace any occurrences ## of relative or full URLs with the respective version. Basically, prefix @@ -59,25 +50,25 @@ fi ## # shellcheck disable=2059 -printf "${COLOR_GREEN}INFO: Replacing relative URLs in $TARGET/$VER for HTML files...${COLOR_RESET}\n" -find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 "$SED" -i -e 's|href="/ee/|href="/'"$VER"'/ee/|g' \ - -e 's|href="/runner/|href="/'"$VER"'/runner/|g' \ - -e 's|href="/omnibus/|href="/'"$VER"'/omnibus/|g' \ - -e 's|href="/charts/|href="/'"$VER"'/charts/|g' \ - -e 's|href="/operator/|href="/'"$VER"'/operator/|g' \ - -e 's|="/assets/|="/'"$VER"'/assets/|g' \ - -e 's|="/frontend/|="/'"$VER"'/frontend/|g' \ - -e 's|||g' \ - -e 's|="/opensearch.xml|="/'"$VER"'/opensearch.xml|g' +printf "${COLOR_GREEN}INFO: Replacing relative URLs in $INPUT for HTML files...${COLOR_RESET}\n" +find "${INPUT}" -type f -name '*.html' -print0 | xargs -0 "$SED" -i -e 's|href="/ee/|href="/'"$VER"'/ee/|g' \ + -e 's|href="/runner/|href="/'"$VER"'/runner/|g' \ + -e 's|href="/omnibus/|href="/'"$VER"'/omnibus/|g' \ + -e 's|href="/charts/|href="/'"$VER"'/charts/|g' \ + -e 's|href="/operator/|href="/'"$VER"'/operator/|g' \ + -e 's|="/assets/|="/'"$VER"'/assets/|g' \ + -e 's|="/frontend/|="/'"$VER"'/frontend/|g' \ + -e 's|||g' \ + -e 's|="/opensearch.xml|="/'"$VER"'/opensearch.xml|g' # shellcheck disable=2059 -printf "${COLOR_GREEN}INFO: Replacing relative URLs in $TARGET/$VER for CSS files...${COLOR_RESET}\n" -find "${TARGET}/$VER" -type f -name '*.css' -print0 | xargs -0 "$SED" -i 's|/assets/|/'"$VER"'/assets/|g' +printf "${COLOR_GREEN}INFO: Replacing relative URLs in $INPUT for CSS files...${COLOR_RESET}\n" +find "${INPUT}" -type f -name '*.css' -print0 | xargs -0 "$SED" -i 's|/assets/|/'"$VER"'/assets/|g' # shellcheck disable=2059 -printf "${COLOR_GREEN}INFO: Replacing relative URLs in $TARGET/$VER for JavaScript files...${COLOR_RESET}\n" -find "${TARGET}/$VER" -type f -name '*.js' -print0 | xargs -0 "$SED" -i -e 's|/search/|/'"$VER"'/search/|g' \ - -e 's|/assets/|/'"$VER"'/assets/|g' +printf "${COLOR_GREEN}INFO: Replacing relative URLs in $INPUT for JavaScript files...${COLOR_RESET}\n" +find "${INPUT}" -type f -name '*.js' -print0 | xargs -0 "$SED" -i -e 's|/search/|/'"$VER"'/search/|g' \ + -e 's|/assets/|/'"$VER"'/assets/|g' # # Full URLs # @@ -88,13 +79,13 @@ find "${TARGET}/$VER" -type f -name '*.js' -print0 | xargs -0 "$SED" -i -e 's|/s # See https://gitlab.com/gitlab-org/gitlab-docs/-/issues/1568 # # shellcheck disable=2059 -printf "${COLOR_GREEN}INFO: Replacing full URLs in $TARGET/$VER for HTML files...${COLOR_RESET}\n" -find "${TARGET}/$VER" -type f -name '*.html' -print0 | xargs -0 "$SED" -i -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/ee/|href="/'"$VER"'/ee/|g' \ - -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/runner/|href="/'"$VER"'/runner/|g' \ - -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/omnibus/|href="/'"$VER"'/omnibus/|g' \ - -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/charts/|href="/'"$VER"'/charts/|g' \ - -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/operator/|href="/'"$VER"'/operator/|g' +printf "${COLOR_GREEN}INFO: Replacing full URLs in $INPUT for HTML files...${COLOR_RESET}\n" +find "${INPUT}" -type f -name '*.html' -print0 | xargs -0 "$SED" -i -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/ee/|href="/'"$VER"'/ee/|g' \ + -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/runner/|href="/'"$VER"'/runner/|g' \ + -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/omnibus/|href="/'"$VER"'/omnibus/|g' \ + -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/charts/|href="/'"$VER"'/charts/|g' \ + -e '/$rel="canonical"\|property="og:url"$/! s|href="https://docs.gitlab.com/operator/|href="/'"$VER"'/operator/|g' # shellcheck disable=2059 printf "${COLOR_GREEN}INFO: Fixing URLs inside the sitemap...${COLOR_RESET}\n" -find "${TARGET}/$VER" -type f -name 'sitemap.xml' -print0 | xargs -0 "$SED" -i 's|docs.gitlab.com/|docs.gitlab.com/'"$VER"'/|g' +find "${INPUT}" -type f -name 'sitemap.xml' -print0 | xargs -0 "$SED" -i 's|docs.gitlab.com/|docs.gitlab.com/'"$VER"'/|g' -- cgit v1.2.3