use the systems number of processes for running static checks - multiple jobs.

4 files changed, 72 insertions, 16 deletions

diff --git a/build_files/cmake/cmake_static_check_cppcheck.py b/build_files/cmake/cmake_static_check_cppcheck.py
index 6eb9d4da8b6..fdd380c5b4d 100644
--- a/build_files/cmake/cmake_static_check_cppcheck.py
+++ b/build_files/cmake/cmake_static_check_cppcheck.py
@@ -37,7 +37,7 @@ CHECKER_BIN = "cppcheck"
 
 CHECKER_ARGS = [
     # not sure why this is needed, but it is.
-    "-I" + os.path.join(project_source_info.SOURCE_DIR, "extern/glew/include"),
+    "-I" + os.path.join(project_source_info.SOURCE_DIR, "extern", "glew", "include"),
 
     #  "--check-config", # when includes are missing
     #  "--enable=all",  # if you want sixty hundred pedantic suggestions
@@ -58,19 +58,21 @@ def main():
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+
+    process_functions = []
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
-        # if percent < 27.9:
-        #    continue
-
-        # let cppcheck finish the line off...
+        sys.stdout.flush()
         sys.stdout.write("%s " % percent_str)
 
-        sys.stdout.flush()
-        process = subprocess.Popen(cmd)
-        process.wait()
+        return subprocess.Popen(cmd)
+
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
+
+    project_source_info.queue_processes(process_functions)
 
 
 if __name__ == "__main__":
diff --git a/build_files/cmake/cmake_static_check_sparse.py b/build_files/cmake/cmake_static_check_sparse.py
index 9af53ed8008..aa2d56e5509 100644
--- a/build_files/cmake/cmake_static_check_sparse.py
+++ b/build_files/cmake/cmake_static_check_sparse.py
@@ -52,15 +52,21 @@ def main():
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
-        sys.stdout.write("%s %s\n" % (percent_str, c))
         sys.stdout.flush()
+        sys.stdout.write("%s %s\n" % (percent_str, c))
+
+        return subprocess.Popen(cmd)
+
+    process_functions = []
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
+
+    project_source_info.queue_processes(process_functions)
 
-        process = subprocess.Popen(cmd)
-        process.wait()
 
 if __name__ == "__main__":
     main()
diff --git a/build_files/cmake/cmake_static_check_splint.py b/build_files/cmake/cmake_static_check_splint.py
index aa13ef82e3c..08d83337426 100644
--- a/build_files/cmake/cmake_static_check_splint.py
+++ b/build_files/cmake/cmake_static_check_splint.py
@@ -84,15 +84,21 @@ def main():
 
         check_commands.append((c, cmd))
 
-    for i, (c, cmd) in enumerate(check_commands):
+    def my_process(i, c, cmd):
         percent = 100.0 * (i / (len(check_commands) - 1))
         percent_str = "[" + ("%.2f]" % percent).rjust(7) + " %:"
 
         sys.stdout.write("%s %s\n" % (percent_str, c))
         sys.stdout.flush()
 
-        process = subprocess.Popen(cmd)
-        process.wait()
+        return subprocess.Popen(cmd)
+
+    process_functions = []
+    for i, (c, cmd) in enumerate(check_commands):
+        process_functions.append((my_process, (i, c, cmd)))
+
+    project_source_info.queue_processes(process_functions)
+
 
 if __name__ == "__main__":
     main()
diff --git a/build_files/cmake/project_source_info.py b/build_files/cmake/project_source_info.py
index 670befbf2aa..83e4e05dc0f 100644
--- a/build_files/cmake/project_source_info.py
+++ b/build_files/cmake/project_source_info.py
@@ -148,6 +148,48 @@ def build_info(use_c=True, use_cxx=True, ignore_prefix_list=None):
     return source
 
 
+# could be moved elsewhere!, this just happens to be used by scripts that also
+# use this module.
+def queue_processes(process_funcs, job_total=-1):
+    """ Takes a list of function arg pairs, each function must return a process
+    """
+    import sys
+
+    if job_total == -1:
+        import multiprocessing
+        job_total = multiprocessing.cpu_count()
+        del multiprocessing
+
+    if job_total == 1:
+        import os
+        import sys
+        for func, args in process_funcs:
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            process = func(*args)
+            process.wait()
+    else:
+        import time
+        import subprocess
+
+        processes = []
+        for func, args in process_funcs:
+            # wait until a thread is free
+            while 1:
+                processes[:] = [p for p in processes if p.poll() is None]
+     
+                if len(processes) <= job_total:
+                    break
+                else:
+                    time.sleep(0.1)
+
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            processes.append(func(*args))
+
+
 def main():
     if not os.path.exists(join(CMAKE_DIR, "CMakeCache.txt")):
         print("This script must run from the cmake build dir")