diff options
author | XapaJIaMnu <nheart@gmail.com> | 2015-05-05 19:36:19 +0300 |
---|---|---|
committer | XapaJIaMnu <nheart@gmail.com> | 2015-05-05 19:36:19 +0300 |
commit | 8e6eb067bca1ee4f9d36cb2c305f7ac60b81f230 (patch) | |
tree | a080bbfe701acea4e704ffc58c4d49dd5a2f6bd0 | |
parent | 2acb5903948688e43decec87ffec5ce9d8a7bd5a (diff) |
Add profiling option to the testsuite. Untested.
-rw-r--r-- | contrib/moses-speedtest/README.md | 18 | ||||
-rw-r--r-- | contrib/moses-speedtest/runtests.py | 89 |
2 files changed, 85 insertions, 22 deletions
diff --git a/contrib/moses-speedtest/README.md b/contrib/moses-speedtest/README.md index c95c6a400..74b7d079e 100644 --- a/contrib/moses-speedtest/README.md +++ b/contrib/moses-speedtest/README.md @@ -27,14 +27,15 @@ DROP_CACHES_COMM: sys_drop_caches 3 TEST_DIR: /home/moses-speedtest/phrase_tables/tests TEST_LOG_DIR: /home/moses-speedtest/phrase_tables/testlogs BASEBRANCH: RELEASE-2.1.1 +MOSES_PROFILER_REPO: /home/moses-speedtest/moses-standard/mosesdecoder-variant-prof </pre> The _MOSES\_REPO\_PATH_ is the place where you have set up and built moses. -The _DROP\_CACHES\_COMM_ is the command that would beused to drop caches. It should run without needing root access. +The _DROP\_CACHES\_COMM_ is the command that would b eused to drop caches. It should run without needing root access. _TEST\_DIR_ is the directory where all the tests will reside. _TEST\_LOG\_DIR_ is the directory where the performance logs will be gathered. It should be created before running the testsuite for the first time. _BASEBRANCH_ is the branch against which all new tests will be compared. It should normally be set to be the latest Moses stable release. - +_MOSES\_PROFILER\_REPO_ is a path to a moses repository set up and built with profiling enabled. Optional if you want to produce profiling results. ### Creating tests In order to create a test one should go into the TEST_DIR and create a new folder. That folder will be used for the name of the test. @@ -44,7 +45,7 @@ An example such configuration file is **test\_config** <pre> Command: moses -f ... -i fff #Looks for the command in the /bin directory of the repo specified in the testsuite_config LDPRE: ldpreloads #Comma separated LD_LIBRARY_PATH:/, -Variants: vanilla, cached, ldpre #Can't have cached without ldpre or vanilla +Variants: vanilla, cached, ldpre, profile #Can't have cached without ldpre or vanilla </pre> The _Command:_ line specifies the executable (which is looked up in the /bin directory of the repo.) and any arguments necessary. Before running the test, the script cds to the current test directory so you can use relative paths. @@ -54,6 +55,17 @@ The _Variants:_ line specifies what type of tests should we run. This particular 2. A vanilla cached test meaning that after the vanilla test, the test will be run again without dropping caches in order to benchmark performance on cached filesystem. 3. A test with LD_PRELOAD ldpreloads moses -f command. For each available LDPRELOAD comma separated library to preload. 4. A cached version of all LD_PRELOAD tests. +5. A profile variant is only available if you have setup the profiler repository. It produces gprof outputs for all of the above in a subdirectory inside the _TEST\_LOG\_DIR. + +#### Produce profiler results. +If you want to produce profiler results together in some tests you need to specify the _MOSES\_PROFILER\_REPO_ in the config +```bash +git clone https://github.com/moses-smt/mosesdecoder.git mosesdecoder-profile +cd mosesdecoder +./bjam -j10 --with-cmph=/usr/include/ variant=profile +``` + +Afterwards for testcases which contain the **profile** keyword in **Variants** you will see a directory inside _TEST\_LOG\_DIR which contains the **gprof** output from every run. ### Running tests. Running the tests is done through the **runtests.py** script. diff --git a/contrib/moses-speedtest/runtests.py b/contrib/moses-speedtest/runtests.py index b82587dce..a043a26db 100644 --- a/contrib/moses-speedtest/runtests.py +++ b/contrib/moses-speedtest/runtests.py @@ -144,7 +144,7 @@ def parse_testconfig(conffile): basebranch = args.replace('\n', '') elif opt == 'BASEREV:': baserev = args.replace('\n', '') - elif opt == 'MOSES_PROFILER_PATH:': # Optional + elif opt == 'MOSES_PROFILER_REPO:': # Optional repo_prof_path = args.replace('\n', '') else: raise ValueError('Unrecognized option ' + opt) @@ -199,9 +199,37 @@ def write_log(time_file, logname, config): log_write.write(writestr) log_write.close() +def write_gprof(command, name, variant, config): + """Produces a gprof report from a gmon file""" + #Check if we have a directory for the profiling of this testcase: + output_dir = config.testlogs + '/' + name + if not os.path.exists(output_dir): + os.makedirs(output_dir) + outputfile = output_dir + '/' + time.strftime("%d.%m.%Y_%H:%M:%S") + '_' + name + '_' + variant + + #Compile a gprof command and output the file in the directory we just created + gmon_path = os.getcwd() + '/gmon.out' # Path to the profiling file + executable_path = command.split(' ')[0] # Path to the moses binary + gprof_command = 'gprof ' + executable_path + ' ' + gmon_path + ' > ' + outputfile + subprocess.call([gprof_command], shell=True) + os.remove('gmon_path') # After we are done discard the gmon file + +def execute_test(command, path, name, variant, config, profile=False): + """Executes a testcase given a whole command, path to the test file output, + name of the test and variant tested. Config is the global configuration""" + subprocess.Popen([command], stdout=None, stderr=subprocess.PIPE, shell=True).communicate() + if not profile: + write_log(path, name + '_' + variant, config) + else: # Basically produce a gmon output + write_gprof(command, name, variant, config) + def execute_tests(testcase, cur_directory, config): """Executes timed tests based on the config file""" + #Several global commands related to the time wrapper + time_command = ' time -p -o /tmp/time_moses_tests ' + time_path = '/tmp/time_moses_tests' + #Figure out the order of which tests must be executed. #Change to the current test directory os.chdir(config.tests + '/' + cur_directory) @@ -211,15 +239,14 @@ def execute_tests(testcase, cur_directory, config): #Perform vanilla test and if a cached test exists - as well print(testcase.name) if 'vanilla' in testcase.permutations: - print(testcase.command) - subprocess.Popen(['time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\ - stderr=subprocess.PIPE, shell=True).communicate() - write_log('/tmp/time_moses_tests', testcase.name + '_vanilla', config) + #Create the command for executing moses + whole_command = time_command + testcase.command + + #test normal and cached + execute_test(whole_command, time_path, testcase.name, 'vanilla', config) if 'cached' in testcase.permutations: - subprocess.Popen(['time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\ - stderr=None, shell=True).communicate() - write_log('/tmp/time_moses_tests', testcase.name + '_vanilla_cached', config) - + execute_test(whole_command, time_path, testcase.name, 'vanilla_cached', config) + #Now perform LD_PRELOAD tests if 'ldpre' in testcase.permutations: for opt in testcase.ldopts: @@ -227,18 +254,42 @@ def execute_tests(testcase, cur_directory, config): subprocess.call(['sync'], shell=True) subprocess.call([config.drop_caches], shell=True) - #test - subprocess.Popen(['LD_PRELOAD ' + opt + ' time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\ - stderr=None, shell=True).communicate() - write_log('/tmp/time_moses_tests', testcase.name + '_ldpre_' + opt, config) + #Create the command for executing moses: + whole_command = 'LD_PRELOAD ' + opt + time_command + testcase.command + variant = 'ldpre_' + opt + + #test normal and cached + execute_test(whole_command, time_path, testcase.name, variant, config) if 'cached' in testcase.permutations: - subprocess.Popen(['LD_PRELOAD ' + opt + ' time -p -o /tmp/time_moses_tests ' + testcase.command], stdout=None,\ - stderr=None, shell=True).communicate() - write_log('/tmp/time_moses_tests', testcase.name + '_ldpre_' +opt +'_cached', config) + execute_test(whole_command, time_path, testcase.name, variant + '_cached', config) + + #Perform profiling test. Mostly same as the above lines but necessary duplication. + #All actual code is inside execute_test so those lines shouldn't need modifying + if 'profile' in testcase.permutations: + subprocess.call(['sync'], shell=True) # Drop caches first + subprocess.call([config.drop_caches], shell=True) + + if 'vanilla' in testcase.permutations: + whole_command = testcase.prof_command + execute_test(whole_command, time_path, testcase.name, 'profile', config, True) + if 'cached' in testcase.permutations: + execute_test(whole_command, time_path, testcase.name, 'profile_cached', config, True) + + if 'ldpre' in testcase.permutations: + for opt in testcase.ldopts: + #Clear caches + subprocess.call(['sync'], shell=True) + subprocess.call([config.drop_caches], shell=True) + + #Create the command for executing moses: + whole_command = 'LD_PRELOAD ' + opt + testcase.prof_command + variant = 'profile_ldpre_' + opt + + #test normal and cached + execute_test(whole_command, time_path, testcase.name, variant, config, True) + if 'cached' in testcase.permutations: + execute_test(whole_command, time_path, testcase.name, variant + '_cached', config, True) - #if 'profile' in testcase.permutations: - #TODO Separate the above into functions so we can execute them with profiling moses. - #Fix the logic in the main # Go through all the test directories and executes tests if __name__ == '__main__': |