CMake: Add support of Ninja's pools to ease building on limited amount of RAM.

Many modern computers support a lot of threads (parrallel building
jobs), but are somewhat restricted in memory, when some building jobs
can require several GB each.

Ninja builder has pools, which extend the usual `-j X` make
parallelizing option, by allowing to specify different numbers of
parallel jobs for different targets.

This commit defines three pools, one for linking, one for usual compile,
and one for compiling some 'heavy' cpp libs, when a single file can
require GB of RAM in full debug builds.

Simply enabling WITH_NINJA_POOL_JOBS will try to set default sensible
values for those three pools based on your machine specifications, you
can then tweak further the values of NINJA_MAX_NUM_PARALLEL_ settings,
if you like.

On my system (8 cores, 16GB RAM), it allows to build a full debug with
all ASAN options build with roughly 7GB of RAM used at most, pretty much
as quickly as without that option (which would require up to 11GB of
available RAM at some points).

Review task: D4780.

1 files changed, 83 insertions, 0 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1cb20142281..f55ae5f5b8a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -565,6 +565,14 @@ if(WIN32)
   mark_as_advanced(WINDOWS_PYTHON_DEBUG)
 endif()
 
+# The following only works with the Ninja generator in CMake >= 3.0.
+if("${CMAKE_GENERATOR}" MATCHES "Ninja")
+  option(WITH_NINJA_POOL_JOBS
+         "Enable Ninja pools of jobs, to try to ease building on machines with 16GB of RAM or less (if not yet defined, will try to set best values based on detected machine specifications)."
+         OFF)
+  mark_as_advanced(WITH_NINJA_POOL_JOBS)
+endif()
+
 # avoid using again
 option_defaults_clear()
 
@@ -1287,6 +1295,76 @@ if(WITH_LIBMV)
 endif()
 
 #-----------------------------------------------------------------------------
+# Extra limits to number of jobs running in parallel for some kind os tasks.
+# Only supported by Ninja build system currently.
+if("${CMAKE_GENERATOR}" MATCHES "Ninja" AND WITH_NINJA_POOL_JOBS)
+  if(NOT NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS AND
+     NOT NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS AND
+     NOT NINJA_MAX_NUM_PARALLEL_LINK_JOBS)
+    # Try to define good default values.
+    # Max mem of heavy cpp files compilation: about 2.5GB
+    # Max mem during linking: about 3.3GB
+    cmake_host_system_information(RESULT _NUM_CORES QUERY NUMBER_OF_LOGICAL_CORES)
+    # Note: this gives mem in MB.
+    cmake_host_system_information(RESULT _TOT_MEM QUERY TOTAL_PHYSICAL_MEMORY)
+
+    # Heuristics... the more cores we have, the more free mem we have to keep for the non-heavy tasks too.
+    if(${_TOT_MEM} LESS 8000 AND ${_NUM_CORES} GREATER 2)
+      set(_compile_heavy_jobs "1")
+    elseif(${_TOT_MEM} LESS 16000 AND ${_NUM_CORES} GREATER 4)
+      set(_compile_heavy_jobs "2")
+    elseif(${_TOT_MEM} LESS 24000 AND ${_NUM_CORES} GREATER 8)
+      set(_compile_heavy_jobs "3")
+    elseif(${_TOT_MEM} LESS 32000 AND ${_NUM_CORES} GREATER 16)
+      set(_compile_heavy_jobs "4")
+    elseif(${_TOT_MEM} LESS 64000 AND ${_NUM_CORES} GREATER 32)
+      set(_compile_heavy_jobs "8")
+    else()
+      set(_compile_heavy_jobs "")
+    endif()
+
+    set(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS "${_compile_heavy_jobs}" CACHE STRING
+        "Define the maximum number of concurrent heavy compilation jobs, for ninja build system (used for some targets which cpp files can take several GB each during compilation)." FORCE)
+    mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS)
+    set(_compile_heavy_jobs)
+
+    # Only set regular compile jobs if we set heavy jobs, otherwise default (using all cores) if fine.
+    if(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS)
+      math(EXPR _compile_jobs "${_NUM_CORES} - 1")
+    else()
+      set(_compile_jobs "")
+    endif()
+    set(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS "${_compile_jobs}" CACHE STRING
+        "Define the maximum number of concurrent compilation jobs, for ninja build system." FORCE)
+    mark_as_advanced(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS)
+    set(_compile_jobs)
+
+    # In practice, even when there is RAM available, this proves to be quicker than running in parallel
+    # (due to slow disks accesses).
+    set(NINJA_MAX_NUM_PARALLEL_LINK_JOBS "1" CACHE STRING
+        "Define the maximum number of concurrent link jobs, for ninja build system." FORCE)
+    mark_as_advanced(NINJA_MAX_NUM_PARALLEL_LINK_JOBS)
+
+    set(_NUM_CORES)
+    set(_TOT_MEM)
+  endif()
+
+  if(NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${NINJA_MAX_NUM_PARALLEL_COMPILE_JOBS})
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+  endif()
+
+  if(NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_heavy_job_pool=${NINJA_MAX_NUM_PARALLEL_COMPILE_HEAVY_JOBS})
+  endif()
+
+  if(NINJA_MAX_NUM_PARALLEL_LINK_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${NINJA_MAX_NUM_PARALLEL_LINK_JOBS})
+    set(CMAKE_JOB_POOL_LINK link_job_pool)
+  endif()
+endif()
+
+#-----------------------------------------------------------------------------
 # Extra compile flags
 
 if(CMAKE_COMPILER_IS_GNUCC)
@@ -1622,6 +1700,11 @@ add_subdirectory(tests)
 
 
 #-----------------------------------------------------------------------------
+# Define 'heavy' submodules (for Ninja builder when using pools).
+setup_heavy_lib_pool()
+
+
+#-----------------------------------------------------------------------------
 # CPack for generating packages
 include(build_files/cmake/packaging.cmake)