Release workflows (#731)

* Add GitHub workflow for generating releases
* Update pattern and job names

1 files changed, 268 insertions, 0 deletions

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 00000000..7cd41220
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,268 @@
+# This workflow builds packages with Marian executables and then attaches them

+# to an existing GitHub release or drafts a new release if needed.

+# There are two ways of utilizing the workflow:

+#

+# 1. Create a new tag locally and push it to the remote, for example:

+#

+#     git tag v1.2.3

+#     git push origin v1.2.3

+#

+# The workflow, if built successfully, will open a new draft release (not seen

+# for the public, available from under the /releases subpage on GitHub), use

+# the last unreleased changes from CHANGELOG as the release description, and

+# attach built packages as assets. Until the draft release is not published,

+# the tag will be separated from the release, but they get merged into a single

+# item with all assets.  An admin user need to manually publish the draft

+# release then. (Note: this behavior can be easily changed if needed).

+#

+# 2. Publish a new (non-draft) release with a tag via GitHub web interface. The

+# workflow, if built successfully, will only attach built packages as

+# additional assets. It is the responsibility of an admin user to add

+# description to the release if it was not provided before..

+#

+name: Release

+

+# The job is triggered by pushing a tag that follows the given pattern name

+on:

+  push:

+    tags:

+      - "v*.*.*"

+

+env:

+  cuda_version: "10.2"

+  gcc_version: 8

+

+jobs:

+  ######################################################################

+  build-release-ubuntu:

+    strategy:

+      matrix:

+        include:

+          # No builds with FBGEMM because they are not portable enough

+          # requiring same CPU architecture at compilation and runtime

+          - name: "Build Ubuntu CPU"

+            suffix: cpu

+            fbgemm: false

+            cuda: false

+          - name: "Build Ubuntu CPU+CUDA"

+            suffix: cuda10.2

+            fbgemm: false

+            cuda: true

+

+    runs-on: ubuntu-18.04

+    name: ${{ matrix.name }}

+

+    steps:

+    # Set env.github_tag_name and env.archive_name for subsequent steps

+    - name: Archive name

+      run: |

+        # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/

+        TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)

+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable

+        echo "::set-env name=github_tag_name::${TAG_NAME}"

+        echo "::set-env name=archive_name::${{ github.event.repository.name }}-${TAG_NAME}_linux-x64-static_${{ matrix.suffix }}"

+      shell: bash

+

+    - name: Checkout

+      uses: actions/checkout@v2

+      with:

+        submodules: recursive

+

+    # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev

+    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev

+    - name: Install dependencies

+      run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler

+

+    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html

+    - name: Install MKL

+      run: |

+        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -

+        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"

+        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"

+        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088

+    # The script simplifies installation of different versions of CUDA

+    - name: Install CUDA

+      run: ./scripts/ci/install_cuda_ubuntu.sh ${{ env.cuda_version }}

+      if: matrix.cuda == true

+

+    # Boost is already installed on GitHub-hosted runners in a non-standard location

+    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671

+    - name: Configure CMake

+      run: |

+        mkdir -p build

+        cd build

+        CC=/usr/bin/gcc-${{ env.gcc_version }} CXX=/usr/bin/g++-${{ env.gcc_version }} CUDAHOSTCXX=/usr/bin/g++-${{ env.gcc_version }} \

+        cmake .. \

+          -DBoost_ARCHITECTURE=-x64 \

+          -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include \

+          -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \

+          -DBOOST_ROOT=$BOOST_ROOT_1_69_0 \

+          -DCMAKE_BUILD_TYPE=Slim \

+          -DCOMPILE_CPU=on \

+          -DCOMPILE_CUDA=${{ matrix.cuda }} \

+          -DCOMPILE_SERVER=on \

+          -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ env.cuda_version }} \

+          -DUSE_FBGEMM=${{ matrix.fbgemm }} \

+          -DUSE_SENTENCEPIECE=on \

+          -DUSE_STATIC_LIBS=on \

+    - name: Compile

+      working-directory: build

+      run: make -j2

+

+    - name: Create archive

+      working-directory: build

+      run: tar zcvf ../${{ env.archive_name }}.tar.gz marian* ../README.md

+

+    # For testing only

+    #- name: Test archive

+      #run: tar zcvf ${{ env.archive_name }}.tar.gz README.md

+

+    - name: Upload archive

+      uses: actions/upload-artifact@v2

+      with:

+        name: ${{ env.archive_name }}.tar.gz

+        path: ${{ env.archive_name }}.tar.gz

+

+  ######################################################################

+  build-release-windows:

+    strategy:

+      matrix:

+        include:

+          # No builds with FBGEMM because they are not portable enough

+          # requiring same CPU architecture at compilation and runtime

+          - name: "Build Windows CPU"

+            suffix: cpu

+            fbgemm: false

+            cuda: false

+          - name: "Build Windows CPU+CUDA"

+            suffix: cuda10.2

+            fbgemm: false

+            cuda: true

+

+    runs-on: windows-2019

+    name: ${{ matrix.name }}

+

+    steps:

+    # Set env.github_tag_name and env.archive_name for subsequent steps

+    - name: Archive name

+      run: |

+        # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/

+        TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-)

+        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable

+        echo "::set-env name=github_tag_name::${TAG_NAME}"

+        echo "::set-env name=archive_name::${{ github.event.repository.name }}-${TAG_NAME}_windows-x64_${{ matrix.suffix }}"

+      shell: bash

+

+    - name: Checkout

+      uses: actions/checkout@v2

+      with:

+        submodules: recursive

+

+    - name: Download MKL

+      run: |

+        C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip

+        Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl

+        # Set the MKLROOT environment variable so that CMake can find MKL.

+        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners

+        echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"

+      shell: powershell

+

+    - name: Install CUDA

+      run: |

+        .\scripts\ci\install_cuda_windows.ps1 '${{ env.cuda_version }}'

+        # Set path to CUDA for subsequent steps so that CMake can find it

+        echo "::set-env name=CUDA_PATH::$env:CUDA_PATH"

+        echo "::add-path::$env:CUDA_PATH/bin"

+      shell: powershell

+      if: matrix.cuda == true

+

+    - name: Prepare vcpkg

+      uses: lukka/run-vcpkg@v2

+      with:

+        vcpkgArguments: protobuf

+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da

+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/

+        vcpkgTriplet: x64-windows-static

+

+    # Build with a simplified CMake settings JSON file

+    - name: Run CMake

+      uses: lukka/run-cmake@v2

+      with:

+        buildDirectory: ${{ github.workspace }}/build/

+        cmakeAppendedArgs: '-G Ninja

+          -DCMAKE_BUILD_TYPE="Release"

+          -DOPENSSL_USE_STATIC_LIBS="TRUE"

+          -DOPENSSL_MSVC_STATIC_RT="TRUE"

+          -DCOMPILE_CPU="TRUE"

+          -DCOMPILE_CUDA="${{ matrix.cuda }}"

+          -DCOMPILE_SERVER="FALSE"

+          -DUSE_FBGEMM="${{ matrix.fbgemm }}"

+          -DUSE_MPI="FALSE"

+          -DUSE_NCCL="FALSE"

+          -DUSE_SENTENCEPIECE="TRUE"

+          -DUSE_STATIC_LIBS="TRUE"'

+        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced

+        cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt

+        useVcpkgToolchainFile: true

+

+    - name: Create archive

+      run: |

+        cp ../README.md .

+        Compress-Archive -Path marian*.exe,README.md -DestinationPath ../${{ env.archive_name }}.zip

+      shell: powershell

+      working-directory: build

+

+    # For testing only

+    #- name: Test archive

+      #run: |

+        #Compress-Archive -Path README.md -DestinationPath ${{ env.archive_name }}.zip

+      #shell: powershell

+

+    - name: Upload archive

+      uses: actions/upload-artifact@v2

+      with:

+        name: ${{ env.archive_name }}.zip

+        path: ${{ env.archive_name }}.zip

+

+  ######################################################################

+  release:

+    needs: [build-release-ubuntu, build-release-windows]

+

+    runs-on: ubuntu-18.04

+    name: Release

+

+    steps:

+    - name: Checkout

+      uses: actions/checkout@v2

+      with:

+        submodules: false  # the repo is needed only for CHANGELOG, so submodules are not needed

+

+    # Extract entire markdown text from the [Unreleased] section to be used as the release description

+    - name: Extract changelog

+      run: |

+        cat CHANGELOG.md | sed -n "/^## \[Unreleased\]/,/^## \[[0-9]/p;/^## \[[0-9]/q" | sed '1d;$d' > RELEASE.md

+        cat RELEASE.md

+    # Downloads all artifacts to the current working directory

+    - name: Download archives

+      uses: actions/download-artifact@v2

+

+    # Artifacts are downloaded into directories with the same names as actual files.

+    # They are moved to a common directory ./assets/

+    - name: Prepare archives

+      run: |

+        mkdir -p assets

+        mv ${{ github.event.repository.name }}-*/* assets/

+        rmdir ${{ github.event.repository.name }}-*

+        ls -lh assets/

+    - name: Release

+      uses: softprops/action-gh-release@v1

+      if: startsWith(github.ref, 'refs/tags/')

+      env:

+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

+      with:

+        body_path: RELEASE.md

+        draft: true

+        files: |

+          assets/*.zip

+          assets/*.tar.gz

+        prerelease: false