Add profiling option to the testsuite. Untested.

2 files changed, 85 insertions, 22 deletions

diff --git a/contrib/moses-speedtest/README.md b/contrib/moses-speedtest/README.md
index c95c6a400..74b7d079e 100644
--- a/contrib/moses-speedtest/README.md
+++ b/contrib/moses-speedtest/README.md
@@ -27,14 +27,15 @@ DROP_CACHES_COMM: sys_drop_caches 3
 TEST_DIR: /home/moses-speedtest/phrase_tables/tests
 TEST_LOG_DIR: /home/moses-speedtest/phrase_tables/testlogs
 BASEBRANCH: RELEASE-2.1.1
+MOSES_PROFILER_REPO: /home/moses-speedtest/moses-standard/mosesdecoder-variant-prof
 </pre>
 
 The _MOSES\_REPO\_PATH_ is the place where you have set up and built moses.
-The _DROP\_CACHES\_COMM_ is the command that would beused to drop caches. It should run without needing root access.
+The _DROP\_CACHES\_COMM_ is the command that would b eused to drop caches. It should run without needing root access.
 _TEST\_DIR_ is the directory where all the tests will reside.
 _TEST\_LOG\_DIR_ is the directory where the performance logs will be gathered. It should be created before running the testsuite for the first time.
 _BASEBRANCH_ is the branch against which all new tests will be compared. It should normally be set to be the latest Moses stable release.
-
+_MOSES\_PROFILER\_REPO_ is a path to a moses repository set up and built with profiling enabled. Optional if you want to produce profiling results.
 ### Creating tests
 
 In order to create a test one should go into the TEST_DIR and create a new folder. That folder will be used for the name of the test.
@@ -44,7 +45,7 @@ An example such configuration file is **test\_config**
 <pre>
 Command: moses -f ... -i fff #Looks for the command in the /bin directory of the repo specified in the testsuite_config
 LDPRE: ldpreloads #Comma separated LD_LIBRARY_PATH:/, 
-Variants: vanilla, cached, ldpre #Can't have cached without ldpre or vanilla
+Variants: vanilla, cached, ldpre, profile #Can't have cached without ldpre or vanilla
 </pre>
 
 The _Command:_ line specifies the executable (which is looked up in the /bin directory of the repo.) and any arguments necessary. Before running the test, the script cds to the current test directory so you can use relative paths.
@@ -54,6 +55,17 @@ The _Variants:_ line specifies what type of tests should we run. This particular
 2. A vanilla cached test meaning that after the vanilla test, the test will be run again without dropping caches in order to benchmark performance on cached filesystem.
 3. A test with LD_PRELOAD ldpreloads moses -f command. For each available LDPRELOAD comma separated library to preload.
 4. A cached version of all LD_PRELOAD tests.
+5. A profile variant is only available if you have setup the profiler repository. It produces gprof outputs for all of the above in a subdirectory inside the _TEST\_LOG\_DIR.
+
+#### Produce profiler results.
+If you want to produce profiler results together in some tests you need to specify the _MOSES\_PROFILER\_REPO_ in the config
+```bash
+git clone https://github.com/moses-smt/mosesdecoder.git mosesdecoder-profile
+cd mosesdecoder
+./bjam -j10 --with-cmph=/usr/include/ variant=profile
+```
+
+Afterwards for testcases which contain the **profile** keyword in **Variants** you will see a directory inside _TEST\_LOG\_DIR which contains the **gprof** output from every run.
 
 ### Running tests.
 Running the tests is done through the **runtests.py** script.
diff --git a/contrib/moses-speedtest/runtests.py b/contrib/moses-speedtest/runtests.py
index b82587dce..a043a26db 100644
--- a/contrib/moses-speedtest/runtests.py
+++ b/contrib/moses-speedtest/runtests.py
@@ -144,7 +144,7 @@ def parse_testconfig(conffile):
             basebranch = args.replace('\n', '')
         elif opt == 'BASEREV:':
             baserev = args.replace('\n', '')
-        elif opt == 'MOSES_PROFILER_PATH:':  # Optional
+        elif opt == 'MOSES_PROFILER_REPO:':  # Optional
             repo_prof_path = args.replace('\n', '')
         else:
             raise ValueError('Unrecognized option ' + opt)
@@ -199,9 +199,37 @@ def write_log(time_file, logname, config):
     log_write.write(writestr)
     log_write.close()
 
+def write_gprof(command, name, variant, config):
+    """Produces a gprof report from a gmon file"""
+    #Check if we have a directory for the profiling of this testcase:
+    output_dir = config.testlogs + '/' + name
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    outputfile = output_dir + '/' + time.strftime("%d.%m.%Y_%H:%M:%S") + '_' + name + '_' + variant
+
+    #Compile a gprof command and output the file in the directory we just created
+    gmon_path = os.getcwd() + '/gmon.out'  # Path to the profiling file
+    executable_path = command.split(' ')[0]  # Path to the moses binary
+    gprof_command = 'gprof ' + executable_path + ' ' + gmon_path + ' > ' + outputfile
+    subprocess.call([gprof_command], shell=True)
+    os.remove('gmon_path')  # After we are done discard the gmon file
+
+def execute_test(command, path, name, variant, config, profile=False):
+    """Executes a testcase given a whole command, path to the test file output,
+    name of the test and variant tested. Config is the global configuration"""
+    subprocess.Popen([command], stdout=None, stderr=subprocess.PIPE, shell=True).communicate()
+    if not profile:
+        write_log(path, name + '_' + variant, config)
+    else:  # Basically produce a gmon output
+        write_gprof(command, name, variant, config)
+
 
 def execute_tests(testcase, cur_directory, config):
     """Executes timed tests based on the config file"""
+    #Several global commands related to the time wrapper
+    time_command = ' time -p -o /tmp/time_moses_tests '
+    time_path = '/tmp/time_moses_tests'
+
     #Figure out the order of which tests must be executed.
     #Change to the current test directory
     os.chdir(config.tests + '/' + cur_directory)
@@ -211,15 +239,14 @@ def execute_tests(testcase, cur_directory, config):
     #Perform vanilla test and if a cached test exists - as well
     print(testcase.name)
     if 'vanilla' in testcase.permutations:
-        print(testcase.command)
-        subprocess.Popen(['time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\
-         stderr=subprocess.PIPE, shell=True).communicate()
-        write_log('/tmp/time_moses_tests', testcase.name + '_vanilla', config)
+        #Create the command for executing moses
+        whole_command = time_command + testcase.command
+
+        #test normal and cached
+        execute_test(whole_command, time_path, testcase.name, 'vanilla', config)
         if 'cached' in testcase.permutations:
-            subprocess.Popen(['time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\
-            stderr=None, shell=True).communicate()
-            write_log('/tmp/time_moses_tests', testcase.name + '_vanilla_cached', config)
-    
+            execute_test(whole_command, time_path, testcase.name, 'vanilla_cached', config)
+
     #Now perform LD_PRELOAD tests
     if 'ldpre' in testcase.permutations:
         for opt in testcase.ldopts:
@@ -227,18 +254,42 @@ def execute_tests(testcase, cur_directory, config):
             subprocess.call(['sync'], shell=True)
             subprocess.call([config.drop_caches], shell=True)
 
-            #test
-            subprocess.Popen(['LD_PRELOAD ' + opt + ' time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\
-            stderr=None, shell=True).communicate()
-            write_log('/tmp/time_moses_tests', testcase.name + '_ldpre_' + opt, config)
+            #Create the command for executing moses:
+            whole_command = 'LD_PRELOAD ' + opt + time_command + testcase.command
+            variant = 'ldpre_' + opt
+
+            #test normal and cached
+            execute_test(whole_command, time_path, testcase.name, variant, config)
             if 'cached' in testcase.permutations:
-                subprocess.Popen(['LD_PRELOAD ' + opt + ' time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\
-                stderr=None, shell=True).communicate()
-                write_log('/tmp/time_moses_tests', testcase.name + '_ldpre_' +opt +'_cached', config)
+                execute_test(whole_command, time_path, testcase.name, variant + '_cached', config)
+
+    #Perform profiling test. Mostly same as the above lines but necessary duplication.
+    #All actual code is inside execute_test so those lines shouldn't need modifying
+    if 'profile' in testcase.permutations:
+        subprocess.call(['sync'], shell=True)  # Drop caches first
+        subprocess.call([config.drop_caches], shell=True)
+
+        if 'vanilla' in testcase.permutations:
+            whole_command = testcase.prof_command
+            execute_test(whole_command, time_path, testcase.name, 'profile', config, True)
+            if 'cached' in testcase.permutations:
+                execute_test(whole_command, time_path, testcase.name, 'profile_cached', config, True)
+
+        if 'ldpre' in testcase.permutations:
+            for opt in testcase.ldopts:
+                #Clear caches
+                subprocess.call(['sync'], shell=True)
+                subprocess.call([config.drop_caches], shell=True)
+
+                #Create the command for executing moses:
+                whole_command = 'LD_PRELOAD ' + opt + testcase.prof_command
+                variant = 'profile_ldpre_' + opt
+
+                #test normal and cached
+                execute_test(whole_command, time_path, testcase.name, variant, config, True)
+                if 'cached' in testcase.permutations:
+                    execute_test(whole_command, time_path, testcase.name, variant + '_cached', config, True)
 
-    #if 'profile' in testcase.permutations:
-        #TODO Separate the above into functions so we can execute them with profiling moses.
-        #Fix the logic in the main
 
 # Go through all the test directories and executes tests
 if __name__ == '__main__':