Merged PR 19904: Update instructions for building on Windows

Changes proposed in this pull request:
1. Clarified instructions how to build Marian on Windows in `vs/README.md`.
2. `vs/CheckOrInstallDeps.bat` does not stop if CUDA or MKL are not installed.

3 files changed, 123 insertions, 144 deletions

diff --git a/vs/CheckOrInstallDeps.bat b/vs/CheckOrInstallDeps.bat
index 2fd1f5b3..65c5e252 100644
--- a/vs/CheckOrInstallDeps.bat
+++ b/vs/CheckOrInstallDeps.bat
@@ -2,7 +2,7 @@
 :: Usage: CheckOrInstallDeps.bat

 ::

 :: This script is used to verify that all the dependencies required to build Marian are available.

-:: The Cuda SDK and the Intel MKL must be installed beforehand by the user.

+:: The CUDA SDK and the Intel MKL must be installed beforehand by the user.

 :: The rest of libraries (see README.md), if not found, will be installed by this script using

 :: vcpkg.

 ::

@@ -96,6 +96,9 @@ echo.
 echo --- Checking dependencies...

 

 set CMAKE_OPT=

+set FOUND_CUDA=

+set FOUND_MKL=

+set FOUND_BOOST=

 

 

 :: -------------------------

@@ -105,8 +108,9 @@ echo.
 echo ... CUDA

 if "%CUDA_PATH%"=="" (

     echo The CUDA_PATH environment variable is not defined: this will compile only the CPU version.

+    set "FOUND_CUDA=false"

 ) else (

-    echo Found Cuda SDK in %CUDA_PATH%

+    echo Found Cuda SDK in "%CUDA_PATH%"

 )

 

 :: -------------------------

@@ -119,26 +123,30 @@ echo ... Intel MKL
 if "%MKLROOT%" == "" (

     set "MKLROOT=C:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl"

 )

+

 if not exist "%MKLROOT%" (

     echo MKLROOT is set to a non existing path:

-    echo "%MKLROOT%"

+    echo     "%MKLROOT%"

     echo Please make sure the Intel MKL libraries are installed and set MKLROOT to the installation path.

-    exit /b 1

-)

-if not exist "%MKLROOT%\include\mkl_version.h" (

+    set "FOUND_MKL=false"

+) else if not exist "%MKLROOT%\include\mkl_version.h" (

     echo MKL header files were not found in this folder:

-    echo    "%MKLROOT%"

+    echo    "%MKLROOT%\include"

     echo Please make sure Intel MKL is properly installed.

-    exit /b 1

-)

-if not exist "%MKLROOT%\lib\intel64\mkl_core.lib" (

+    set "FOUND_MKL=false"

+) else if not exist "%MKLROOT%\lib\intel64\mkl_core.lib" (

     echo MKL library files were not found in this folder:

-    echo    "%MKLROOT%"

+    echo    "%MKLROOT%\lib\intel64"

     echo Please make sure Intel MKL is properly installed.

-    exit /b 1

+    set "FOUND_MKL=false"

+) else (

+    echo Found Intel MKL library in "%MKLROOT%"

 )

 

-echo Found Intel MKL library in %MKLROOT%

+if "%FOUND_MKL%" == "false" if "%FOUND_CUDA%" == "false" (

+	echo.

+	echo Error: neither CUDA SDK nor Intel MKL were found, but at least one of them must be installed.

+)

 

 :: -------------------------

 :: BOOST_INCLUDEDIR and BOOST_LIBRARYDIR can be both set to an existing Boost installation.

@@ -156,29 +164,31 @@ if not exist "%BOOST_INCLUDEDIR%" (
     echo BOOST_INCLUDEDIR is set to a non existing path:

     echo    "%BOOST_INCLUDEDIR%"

     echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.

-    exit /b 1

-)

-if not exist "%BOOST_INCLUDEDIR%\boost\version.hpp" (

+    set "FOUND_BOOST=false"

+) else if not exist "%BOOST_INCLUDEDIR%\boost\version.hpp" (

     echo Boost header files were not found in this folder:

-    echo    "%BOOST_INCLUDEDIR%"

+    echo    "%BOOST_INCLUDEDIR%\boost"

     echo Please make sure Boost is correctly installed.

-    exit /b 1

-)

-

-if not exist "%BOOST_LIBRARYDIR%" (

+    set "FOUND_BOOST=false"

+) else if not exist "%BOOST_LIBRARYDIR%" (

     echo BOOST_LIBRARYDIR is set to a non existing path:

     echo    "%BOOST_LIBRARYDIR%"

     echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.

-    exit /b 1

-)

-if not exist "%BOOST_LIBRARYDIR%\boost_*.lib" (

+    set "FOUND_BOOST=false"

+) else if not exist "%BOOST_LIBRARYDIR%\boost_*.lib" (

     echo Boost library files were not found in this folder:

     echo    "%BOOST_LIBRARYDIR%"

     echo Please make sure Boost is correctly installed.

-    exit /b 1

+    set "FOUND_BOOST=false"

+) else (

+    echo Found Boost headers in "%BOOST_INCLUDEDIR%" and libs in "%BOOST_LIBRARYDIR%"

+)

+

+if "%FOUND_BOOST%" == "false" (

+    echo.

+    echo Warning: Boost was not found. marian-server will not be compiled.

 )

 

-echo Found Boost headers in "%BOOST_INCLUDEDIR%" and libs in "%BOOST_LIBRARYDIR%"

 

 :: -------------------------

 :: OPENSSL_ROOT_DIR can be set to an existing OpenSSL installation.

diff --git a/vs/NOTES.md b/vs/NOTES.md
index 4d00fab5..fb6195e6 100644
--- a/vs/NOTES.md
+++ b/vs/NOTES.md
@@ -1,6 +1,35 @@
 # How to build Marian on Windows with GPU support

 

-This is interesting for developers, exctracted from README.

+This is interesting mostly for developers. Warning: it has been extracted from

+an old `vs/README.md` and some information might be outdated.

+

+---

+## Known issues

+

+1. __Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__

+

+    When using CUDA 9.2, the latest versions of Visual Studio 2017 are not

+    officially supported by CUDA. Two fixes are proposed:

+    - Downgrade Visual Studio to a supported version

+    - Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:

+

+            131     #if _MSC_VER < 1600 || _MSC_VER > 1914

+

+        into:

+

+            131     #if _MSC_VER < 1600 || _MSC_VER > 1915

+

+    For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)

+

+2. __It does not compile with Boost 1.73 or newer__

+

+    It may happen that SimpleWebSocketServer, a 3rd party library that Marian uses for

+    marian-server, does not support the version of Boost available in vcpkg. In such case install a

+    supported version of Boost; if you use vcpkg, an option is to checkout to #5970385, which has

+    Boost 1.72.

+

+    Note that Boost is required only if you compile with marian-server, for compilation using CMake,

+    it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.

 

 ---

 ## Changes from the master branch

diff --git a/vs/README.md b/vs/README.md
index c445f99f..e3a5ee35 100644
--- a/vs/README.md
+++ b/vs/README.md
@@ -1,34 +1,34 @@
-# How to build Marian on Windows with GPU support

+# Building Marian on Windows

 

 

 ## Install prerequisites

 

-The following SDK are required to build Marian with GPU support. At least one of them needs to be

-installed. If only CUDA is installed but not MKL, a GPU-only version will be build. If only MKL is

-installed and not CUDA, only the CPU version will be built. So if you are interested in only one

-functionality, you can omit one of them. Install both for full functionality.

+At least one of the following SDK is required to build Marian on Windows:

 

    - [CUDA](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal),

-     Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA 9.2, see below

+     Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA

+     9.2, see below

    - [Intel MKL](https://software.intel.com/en-us/mkl)

 

+CUDA is required for running Marian on GPU, and Intel MKL for CPU. If only one

+of them is installed, a GPU-only or CPU-only version can be built.

+

 ---

 ## Check dependencies : `CheckOrInstallDeps.bat`

 

-In addition to the 2 previous prerequisites, Marian may need the following libraries that you may

-already have on your system:

+The script `CheckOrInstallDeps.bat` can be used to verify that all dependencies

+are found on your system. If not, it will use the `vcpkg` library manager to

+download and manage your dependencies for CMake, including the following

+optional libraries needed only if you want to compile Marian server:

 

-    - Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in CMake)

+    - Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in

+      `CMakeSettings.json`)

     - OpenSSL, optional for marian-server

 

-The script `CheckOrInstallDeps.bat` can be used to verify that all dependencies are found on your

-system. If not, it will use the `vcpkg` library manager to download and manage your dependencies for

-CMake.

-

-If you already have a working `vcpkg` installation, this script can use it.

-If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.

-Otherwise, you need to edit the script and set the `VCPKG_ROOT` variable.

-Please see the script for more details.

+If you already have a working `vcpkg` installation, this script can use it. If

+vcpkg is in your `PATH` environment variable, the script will find it and use

+it automatically. Otherwise, you need to edit the script and set the

+`VCPKG_ROOT` variable. Please see the script for more details.

 

 ---

 ## Build the project

@@ -41,119 +41,59 @@ There are 3 alternatives to build the project:
 

 ### 1. Use VS2017+ with built-in support for CMake

 

-VS2017 or newer now allows to develop projects built with CMake without the need to generate VS

-projects and solutions. For more information, please read [this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/)

+VS2017 or newer now allows to develop projects built with CMake without the

+need to generate VS projects and solutions. For more information, please read

+[this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/)

 from the Visual C++ Team.

 

-You just need to open the root folder of the git repository in VS (which contains the file

-`CMakeSettings.json`):

-

-- In an Explorer window, right-click then `Open in Visual Studio`

-- In a VS2017 instance, `File > Open > Folder...`

-

-You may need to edit the file `CMakeSettings.json` to set the environment variable for the

-dependencies.

-

-The developing experience is very similar that when using a solution file (Intellisense, build

-project with `F7`, debug, set breakpoints and watch variables, ...), except that the project

-configuration is done in 3 different files:

-

-   - `CMakeList.txt`: this is the CMake source file from the original project.

-     It is used to configure the build targets, add/remove files to compile and configure the

-     compiler flags.

-

-   - `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.

-     Use this file to configure the environment variables and the parameters passed to CMake to

-     generate the project.

-

-   - `.vs\launch.vs.json`: this is an optional user specific file and it is not commited in the Git

-     repo. Use this file to configure the debugging targets. For example:

-

-         {

-             "version": "0.2.1",

-             "defaults": {},

-             "configurations": [

-                 {

-                 "type": "default",

-                 "name": "Training Basics",

-                 "project": "CMakeLists.txt",

-                 "projectTarget": "marian.exe",

-                 "currentDir": "D:\\Perso\\github\\marian\\marian-examples\\training-basics",

-                 "args": [

-                     "--devices 0",

-                     "--type amun",

-                     "--model model/model.npz",

-                     "--train-sets data/corpus.bpe.ro data/corpus.bpe.en",

-                     "--vocabs model/vocab.ro.yml model/vocab.en.yml",

-                     "--dim-vocabs 66000 50000",

-                     "--mini-batch-fit",

-                     "-w 3000",

-                     "--layer-normalization",

-                     "--dropout-rnn 0.2",

-                     "--dropout-src 0.1",

-                     "--dropout-trg 0.1",

-                     "--early-stopping 5",

-                     "--valid-freq 100",

-                     "--save-freq 10000",

-                     "--disp-freq 100",

-                     "--valid-metrics cross-entropy translation",

-                     "--valid-sets data/newsdev2016.bpe.ro data/newsdev2016.bpe.en",

-                     "--valid-script-path .\\scripts\\validate.bat",

-                     "--log model/train.log",

-                     "--valid-log model/valid.log",

-                     "--overwrite",

-                     "--keep-best",

-                     "--seed 1111",

-                     "--exponential-smoothing",

-                     "--normalize 1",

-                     "--beam-size 12",

-                     "--quiet-translation"

-                 ]

-                 }

-             ]

-         }

-

-

-### 2. Create solution and projects files for Visual Studio : `CreateVSProjects.bat`

-

-If you have a previous version of Visual Studio, you will need to use CMake to generate the projects

-files.

+1. Open the root folder of the git repository in VS (which contains the file

+   `CMakeSettings.json`) using `Open local folder` on the welcome page or `File

+   > Open > Folder...` in a VS instance.

+2. Edit the file `CMakeSettings.json` to set the environment variable for the

+   dependencies. Set `COMPILE_CUDA` or `COMPILE_CPU` to `FALSE` if you wish to

+   compile a CPU-only or a GPU-only version respectively.

+3. VS2017 should automatically detect `CMakeSettings.json` and generate CMake

+   Cache.

+4. Build the project with `F7`. If build is successful, the executables will be

+   in the `build` folder.

 

-The provided script `CreateVSProjects.bat` runs the dependency checks then invokes CMake with the

-right parameters to create the solutions for Visual Studio.

 

+#### Development

 

-### 3. Use MSBuild : `BuildRelease.bat`

+The developing experience is very similar that when using a solution file

+(Intellisense, build project with `F7`, debug, set breakpoints and watch

+variables, ...), except that the project configuration is done in 3 different

+files:

 

-The last alternative is to use the script `BuildRelease.bat` that will:

-- Check the dependencies

-- Create the VS project files

-- Invoke MSBuild on these projects to build the targets in Release.

+- `CMakeList.txt`: this is the CMake source file from the original project.

+  It is used to configure the build targets, add/remove files to compile and configure the

+  compiler flags.

 

----

-## Known issues

+- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.

+  Use this file to configure the environment variables and the parameters passed to CMake to

+  generate the project.

 

-1. __Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__

+- `.vs\launch.vs.json`: this is an optional user specific file and it is not commited in the Git

+  repo. Use this file to configure the debugging targets.

 

-    When using CUDA 9.2, the latest versions of Visual Studio 2017 are not officially supported by

-    CUDA. Two fixes are proposed:

-    - Downgrade Visual Studio to a supported version

-    - Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:

 

-            131     #if _MSC_VER < 1600 || _MSC_VER > 1914

+### 2. Create solution and projects files for Visual Studio : `CreateVSProjects.bat`

 

-        into:

+If you have a previous version of Visual Studio, you will need to use CMake to

+generate the projects

+files.

 

-            131     #if _MSC_VER < 1600 || _MSC_VER > 1915

+The provided script `CreateVSProjects.bat` runs the dependency checks then

+invokes CMake with the right parameters to create the solutions for Visual

+Studio.

 

-    For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)

+Warning: the Visual Studio Solution file included in the `vs/` folder might not

+work out of the box with your environment and require customization.

 

-2. __It does not compile with Boost 1.73 or newer__

 

-    It may happen that SimpleWebSocketServer, a 3rd party library that Marian uses for

-    marian-server, does not support the version of Boost available in vcpkg. In such case install a

-    supported version of Boost; if you use vcpkg, an option is to checkout to #5970385, which has

-    Boost 1.72.

+### 3. Use MSBuild : `BuildRelease.bat`

 

-    Note that Boost is required only if you compile with marian-server, for compilation using CMake,

-    it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.

+The last alternative is to use the script `BuildRelease.bat` that will:

+- Check the dependencies.

+- Create the VS project files.

+- Invoke MSBuild on these projects to build the targets in Release.