Add GitHub workflows with CUDA and MKL (#700)

* Add GitHub workflows with CUDA
* Install MKL on Windows
* Upload CPU-only executables from Windows and Ubuntu as artifacts
* Clean up workflows

6 files changed, 228 insertions, 53 deletions

diff --git a/.github/workflows/build-windows-2019-cpu.yml b/.github/workflows/build-windows-2019-cpu.yml
deleted file mode 100644
index 13ef6614..00000000
--- a/.github/workflows/build-windows-2019-cpu.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-name: windows-2019-cpu
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-
-    runs-on: windows-2019
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    - name: Prepare vcpkg
-      uses: lukka/run-vcpkg@v3
-      with:
-        vcpkgArguments: protobuf
-        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
-        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
-        vcpkgTriplet: x64-windows-static
-
-    # Note that we build with a simplified CMake settings JSON file
-    - name: Run CMake
-      uses: lukka/run-cmake@v2
-      with:
-        buildDirectory: ${{ github.workspace }}/build/
-        cmakeAppendedArgs: -G Ninja
-        cmakeListsOrSettingsJson: CMakeSettingsJson
-        cmakeSettingsJsonPath: ${{ github.workspace }}/CMakeSettingsCI.json
-        useVcpkgToolchainFile: true
-
-    - name: Run unit tests
-      working-directory: build/Debug/
-      run: ctest
-
-    - name: Print versions
-      working-directory: build/Debug/
-      run: |
-        .\marian.exe --version
-        .\marian-decoder.exe --version
-        .\marian-scorer.exe --version
-        .\spm_encode.exe --version
-
diff --git a/.github/workflows/build-macos-10.15-cpu.yml b/.github/workflows/macos-cpu.yml
index 824915b8..07f2a9dd 100644
--- a/.github/workflows/build-macos-10.15-cpu.yml
+++ b/.github/workflows/macos-cpu.yml
@@ -1,4 +1,4 @@
-name: macos-10.5-cpu
+name: macOS CPU-only
 
 on:
   push:
diff --git a/.github/workflows/build-ubuntu-18.04-cpu.yml b/.github/workflows/ubuntu-cpu.yml
index 0b37e4f4..4217053c 100644
--- a/.github/workflows/build-ubuntu-18.04-cpu.yml
+++ b/.github/workflows/ubuntu-cpu.yml
@@ -1,4 +1,4 @@
-name: ubuntu-18.04-cpu
+name: Ubuntu 18.04 CPU-only
 
 on:
   push:
@@ -7,7 +7,7 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
+  build-and-upload:
 
     runs-on: ubuntu-18.04
 
@@ -41,7 +41,8 @@ jobs:
       run: |
         mkdir -p build
         cd build
-        cmake .. -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+        cmake .. -DCMAKE_BUILD_TYPE=Release \
+          -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
           -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
           -DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
           -DBoost_ARCHITECTURE=-x64
@@ -62,3 +63,13 @@ jobs:
         ./marian-scorer --version
         ./spm_encode --version
 
+    - name: Prepare archive
+      working-directory: build
+      run: tar zcvf marian-ubuntu-release-static.tar.gz marian*
+
+    # Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
+    - name: Upload archive
+      uses: actions/upload-artifact@v2
+      with:
+        name: marian-ubuntu-release-static.tar.gz
+        path: build/marian-ubuntu-release-static.tar.gz
diff --git a/.github/workflows/ubuntu-gpu.yml b/.github/workflows/ubuntu-gpu.yml
new file mode 100644
index 00000000..ccea606b
--- /dev/null
+++ b/.github/workflows/ubuntu-gpu.yml
@@ -0,0 +1,84 @@
+name: Ubuntu CPU+CUDA
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        include:
+          # Ubuntu 20.04 supports CUDA 11+
+          # Ubuntu 18.04 supports CUDA 10.1+
+          - os: ubuntu-18.04
+            cuda: "10.2"
+            gcc: 8
+          # Ubuntu 16.04 supports CUDA 8+
+          - os: ubuntu-16.04
+            cuda: "10.2"
+            gcc: 7
+          - os: ubuntu-16.04
+            cuda: 9.2
+            gcc: 7
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
+    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
+    - name: Install dependencies
+      run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
+
+    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+    - name: Install MKL
+      run: |
+        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
+        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
+
+    # The script simplifies installation of different versions of CUDA
+    - name: Install CUDA
+      run: ./scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }}
+
+    # Boost is already installed on GitHub-hosted runners in a non-standard location
+    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
+    - name: Configure CMake
+      run: |
+        mkdir -p build
+        cd build
+        CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
+        cmake .. \
+          -DCOMPILE_CPU=on -DCOMPILE_CUDA=on -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+          -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
+          -DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
+          -DBoost_ARCHITECTURE=-x64 \
+          -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }}
+
+    - name: Compile
+      working-directory: build
+      run: make -j2
+
+    # Unit tests are not run because GitHub-hosted runners do not have GPUs
+    # TODO: add a flag to CMake to compile unit tests only on CPU
+    #- name: Run unit tests
+      #working-directory: build
+      #run: make test
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./marian --version
+        ./marian-decoder --version
+        ./marian-scorer --version
+        ./spm_encode --version
+
diff --git a/.github/workflows/windows-cpu.yml b/.github/workflows/windows-cpu.yml
new file mode 100644
index 00000000..b6543d0d
--- /dev/null
+++ b/.github/workflows/windows-cpu.yml
@@ -0,0 +1,66 @@
+name: Windows CPU-only
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build-and-upload:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Download MKL
+      run: |
+        Invoke-WebRequest -Uri https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -TimeoutSec 600 -OutFile mkl.zip
+        Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
+        # Set MKLROOT environment variables so that CMake can find MKL.
+        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
+        echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
+      shell: powershell
+
+    - name: Prepare vcpkg
+      uses: lukka/run-vcpkg@v2
+      with:
+        vcpkgArguments: protobuf
+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
+        vcpkgTriplet: x64-windows-static
+
+    # Build with a simplified CMake settings JSON file
+    - name: Run CMake
+      uses: lukka/run-cmake@v2
+      with:
+        buildDirectory: ${{ github.workspace }}/build/
+        cmakeAppendedArgs: -G Ninja
+        cmakeListsOrSettingsJson: CMakeSettingsJson
+        # JSON file must be in the same directory as the main CMakeLists.txt
+        cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_CPU.json
+        useVcpkgToolchainFile: true
+
+    - name: Run unit tests
+      working-directory: build/Release/
+      run: ctest
+
+    #- name: Print versions
+      #working-directory: build/Release/
+      #run: |
+        #.\marian.exe --version
+        #.\marian-decoder.exe --version
+        #.\marian-scorer.exe --version
+        #.\spm_encode.exe --version
+      #shell: powershell
+
+    # Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
+    - name: Upload archive
+      uses: actions/upload-artifact@v2
+      with:
+        name: marian-windows-release-static
+        path: build/Release/marian*.exe
diff --git a/.github/workflows/windows-gpu.yml b/.github/workflows/windows-gpu.yml
new file mode 100644
index 00000000..360b64f4
--- /dev/null
+++ b/.github/workflows/windows-gpu.yml
@@ -0,0 +1,63 @@
+name: Windows CPU+CUDA
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install CUDA
+      run: |
+        .\scripts\ci\install_cuda_windows.ps1 "10.2"
+        # Set path to CUDA for subsequent steps so that CMake can find it
+        echo "::set-env name=CUDA_PATH::$env:CUDA_PATH"
+        echo "::add-path::$env:CUDA_PATH/bin"
+      shell: powershell
+
+    - name: Download MKL
+      run: |
+        Invoke-WebRequest -Uri https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -TimeoutSec 600 -OutFile mkl.zip
+        Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
+        # Set MKLROOT environment variables so that CMake can find MKL.
+        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
+        echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
+      shell: powershell
+
+    - name: Prepare vcpkg
+      uses: lukka/run-vcpkg@v2
+      with:
+        vcpkgArguments: protobuf
+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
+        vcpkgTriplet: x64-windows-static
+
+    # Build with a simplified CMake settings JSON file.
+    # On Windows+CUDA we compile with COMPILE_CUDA=on and USE_NCCL=off
+    - name: Run CMake
+      uses: lukka/run-cmake@v2
+      with:
+        buildDirectory: ${{ github.workspace }}/build/
+        cmakeAppendedArgs: -G Ninja
+        cmakeListsOrSettingsJson: CMakeSettingsJson
+        # JSON file must be in the same directory as the main CMakeLists.txt
+        cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_GPU.json
+        useVcpkgToolchainFile: true
+
+    - name: Print versions
+      working-directory: build/Debug/
+      run: |
+        .\marian.exe --version
+        .\marian-decoder.exe --version
+        .\marian-scorer.exe --version
+        .\spm_encode.exe --version