From 97f094b44498e6eceb6925c879e488c3cae3fc03 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Tue, 15 Sep 2020 17:42:37 +0100 Subject: Release workflows (#731) * Add GitHub workflow for generating releases * Update pattern and job names --- .github/workflows/release.yml | 268 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..7cd41220 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,268 @@ +# This workflow builds packages with Marian executables and then attaches them +# to an existing GitHub release or drafts a new release if needed. +# There are two ways of utilizing the workflow: +# +# 1. Create a new tag locally and push it to the remote, for example: +# +# git tag v1.2.3 +# git push origin v1.2.3 +# +# The workflow, if built successfully, will open a new draft release (not seen +# for the public, available from under the /releases subpage on GitHub), use +# the last unreleased changes from CHANGELOG as the release description, and +# attach built packages as assets. Until the draft release is not published, +# the tag will be separated from the release, but they get merged into a single +# item with all assets. An admin user need to manually publish the draft +# release then. (Note: this behavior can be easily changed if needed). +# +# 2. Publish a new (non-draft) release with a tag via GitHub web interface. The +# workflow, if built successfully, will only attach built packages as +# additional assets. It is the responsibility of an admin user to add +# description to the release if it was not provided before.. +# +name: Release + +# The job is triggered by pushing a tag that follows the given pattern name +on: + push: + tags: + - "v*.*.*" + +env: + cuda_version: "10.2" + gcc_version: 8 + +jobs: + ###################################################################### + build-release-ubuntu: + strategy: + matrix: + include: + # No builds with FBGEMM because they are not portable enough + # requiring same CPU architecture at compilation and runtime + - name: "Build Ubuntu CPU" + suffix: cpu + fbgemm: false + cuda: false + - name: "Build Ubuntu CPU+CUDA" + suffix: cuda10.2 + fbgemm: false + cuda: true + + runs-on: ubuntu-18.04 + name: ${{ matrix.name }} + + steps: + # Set env.github_tag_name and env.archive_name for subsequent steps + - name: Archive name + run: | + # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/ + TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-) + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + echo "::set-env name=github_tag_name::${TAG_NAME}" + echo "::set-env name=archive_name::${{ github.event.repository.name }}-${TAG_NAME}_linux-x64-static_${{ matrix.suffix }}" + shell: bash + + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev + # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev + - name: Install dependencies + run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler + + # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html + - name: Install MKL + run: | + wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add - + sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list" + sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088 + # The script simplifies installation of different versions of CUDA + - name: Install CUDA + run: ./scripts/ci/install_cuda_ubuntu.sh ${{ env.cuda_version }} + if: matrix.cuda == true + + # Boost is already installed on GitHub-hosted runners in a non-standard location + # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671 + - name: Configure CMake + run: | + mkdir -p build + cd build + CC=/usr/bin/gcc-${{ env.gcc_version }} CXX=/usr/bin/g++-${{ env.gcc_version }} CUDAHOSTCXX=/usr/bin/g++-${{ env.gcc_version }} \ + cmake .. \ + -DBoost_ARCHITECTURE=-x64 \ + -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include \ + -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \ + -DBOOST_ROOT=$BOOST_ROOT_1_69_0 \ + -DCMAKE_BUILD_TYPE=Slim \ + -DCOMPILE_CPU=on \ + -DCOMPILE_CUDA=${{ matrix.cuda }} \ + -DCOMPILE_SERVER=on \ + -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ env.cuda_version }} \ + -DUSE_FBGEMM=${{ matrix.fbgemm }} \ + -DUSE_SENTENCEPIECE=on \ + -DUSE_STATIC_LIBS=on \ + - name: Compile + working-directory: build + run: make -j2 + + - name: Create archive + working-directory: build + run: tar zcvf ../${{ env.archive_name }}.tar.gz marian* ../README.md + + # For testing only + #- name: Test archive + #run: tar zcvf ${{ env.archive_name }}.tar.gz README.md + + - name: Upload archive + uses: actions/upload-artifact@v2 + with: + name: ${{ env.archive_name }}.tar.gz + path: ${{ env.archive_name }}.tar.gz + + ###################################################################### + build-release-windows: + strategy: + matrix: + include: + # No builds with FBGEMM because they are not portable enough + # requiring same CPU architecture at compilation and runtime + - name: "Build Windows CPU" + suffix: cpu + fbgemm: false + cuda: false + - name: "Build Windows CPU+CUDA" + suffix: cuda10.2 + fbgemm: false + cuda: true + + runs-on: windows-2019 + name: ${{ matrix.name }} + + steps: + # Set env.github_tag_name and env.archive_name for subsequent steps + - name: Archive name + run: | + # Get the tag name only to use it in the archive name. The variable github.ref can not be used because it starts with refs/tags/ + TAG_NAME=$(echo ${{ github.ref }} | cut -d/ -f3-) + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + echo "::set-env name=github_tag_name::${TAG_NAME}" + echo "::set-env name=archive_name::${{ github.event.repository.name }}-${TAG_NAME}_windows-x64_${{ matrix.suffix }}" + shell: bash + + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Download MKL + run: | + C:\msys64\usr\bin\wget.exe -nv https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -O mkl.zip + Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl + # Set the MKLROOT environment variable so that CMake can find MKL. + # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners + echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl" + shell: powershell + + - name: Install CUDA + run: | + .\scripts\ci\install_cuda_windows.ps1 '${{ env.cuda_version }}' + # Set path to CUDA for subsequent steps so that CMake can find it + echo "::set-env name=CUDA_PATH::$env:CUDA_PATH" + echo "::add-path::$env:CUDA_PATH/bin" + shell: powershell + if: matrix.cuda == true + + - name: Prepare vcpkg + uses: lukka/run-vcpkg@v2 + with: + vcpkgArguments: protobuf + vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da + vcpkgDirectory: ${{ github.workspace }}/vcpkg/ + vcpkgTriplet: x64-windows-static + + # Build with a simplified CMake settings JSON file + - name: Run CMake + uses: lukka/run-cmake@v2 + with: + buildDirectory: ${{ github.workspace }}/build/ + cmakeAppendedArgs: '-G Ninja + -DCMAKE_BUILD_TYPE="Release" + -DOPENSSL_USE_STATIC_LIBS="TRUE" + -DOPENSSL_MSVC_STATIC_RT="TRUE" + -DCOMPILE_CPU="TRUE" + -DCOMPILE_CUDA="${{ matrix.cuda }}" + -DCOMPILE_SERVER="FALSE" + -DUSE_FBGEMM="${{ matrix.fbgemm }}" + -DUSE_MPI="FALSE" + -DUSE_NCCL="FALSE" + -DUSE_SENTENCEPIECE="TRUE" + -DUSE_STATIC_LIBS="TRUE"' + cmakeListsOrSettingsJson: CMakeListsTxtAdvanced + cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt + useVcpkgToolchainFile: true + + - name: Create archive + run: | + cp ../README.md . + Compress-Archive -Path marian*.exe,README.md -DestinationPath ../${{ env.archive_name }}.zip + shell: powershell + working-directory: build + + # For testing only + #- name: Test archive + #run: | + #Compress-Archive -Path README.md -DestinationPath ${{ env.archive_name }}.zip + #shell: powershell + + - name: Upload archive + uses: actions/upload-artifact@v2 + with: + name: ${{ env.archive_name }}.zip + path: ${{ env.archive_name }}.zip + + ###################################################################### + release: + needs: [build-release-ubuntu, build-release-windows] + + runs-on: ubuntu-18.04 + name: Release + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: false # the repo is needed only for CHANGELOG, so submodules are not needed + + # Extract entire markdown text from the [Unreleased] section to be used as the release description + - name: Extract changelog + run: | + cat CHANGELOG.md | sed -n "/^## \[Unreleased\]/,/^## \[[0-9]/p;/^## \[[0-9]/q" | sed '1d;$d' > RELEASE.md + cat RELEASE.md + # Downloads all artifacts to the current working directory + - name: Download archives + uses: actions/download-artifact@v2 + + # Artifacts are downloaded into directories with the same names as actual files. + # They are moved to a common directory ./assets/ + - name: Prepare archives + run: | + mkdir -p assets + mv ${{ github.event.repository.name }}-*/* assets/ + rmdir ${{ github.event.repository.name }}-* + ls -lh assets/ + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + body_path: RELEASE.md + draft: true + files: | + assets/*.zip + assets/*.tar.gz + prerelease: false -- cgit v1.2.3