1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
|
"""Given a config file, runs tests"""
import os
import subprocess
import time
from argparse import ArgumentParser
from testsuite_common import processLogLine
def parse_cmd():
"""Parse the command line arguments"""
description = "A python based speedtest suite for moses."
parser = ArgumentParser(description=description)
parser.add_argument("-c", "--configfile", action="store",\
dest="configfile", required=True,\
help="Specify test config file")
parser.add_argument("-s", "--singletest", action="store",\
dest="singletestdir", default=None,\
help="Single test name directory. Specify directory name,\
not full path!")
parser.add_argument("-r", "--revision", action="store",\
dest="revision", default=None,\
help="Specify a specific revison for the test.")
parser.add_argument("-b", "--branch", action="store",\
dest="branch", default=None,\
help="Specify a branch for the test.")
arguments = parser.parse_args()
return arguments
def repoinit(testconfig, profiler=True):
"""Determines revision and sets up the repo. If given the profiler optional
argument, wil init the profiler repo instead of the default one."""
revision = ''
#Update the repo
if profiler:
if testconfig.repo_prof is not None:
os.chdir(testconfig.repo_prof)
else:
raise ValueError('Profiling repo is not defined')
else:
os.chdir(testconfig.repo)
#Checkout specific branch, else maintain main branch
if testconfig.branch != 'master':
subprocess.call(['git', 'checkout', testconfig.branch])
rev, _ = subprocess.Popen(['git', 'rev-parse', 'HEAD'],\
stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
revision = str(rev).replace("\\n'", '').replace("b'", '')
else:
subprocess.call(['git checkout master'], shell=True)
#Check a specific revision. Else checkout master.
if testconfig.revision:
subprocess.call(['git', 'checkout', testconfig.revision])
revision = testconfig.revision
elif testconfig.branch == 'master':
subprocess.call(['git pull'], shell=True)
rev, _ = subprocess.Popen(['git rev-parse HEAD'], stdout=subprocess.PIPE,\
stderr=subprocess.PIPE, shell=True).communicate()
revision = str(rev).replace("\\n'", '').replace("b'", '')
return revision
class Configuration:
"""A simple class to hold all of the configuration constatns"""
def __init__(self, repo, drop_caches, tests, testlogs, basebranch, baserev, repo_prof=None):
self.repo = repo
self.repo_prof = repo_prof
self.drop_caches = drop_caches
self.tests = tests
self.testlogs = testlogs
self.basebranch = basebranch
self.baserev = baserev
self.singletest = None
self.revision = None
self.branch = 'master' # Default branch
def additional_args(self, singletest, revision, branch):
"""Additional configuration from command line arguments"""
self.singletest = singletest
if revision is not None:
self.revision = revision
if branch is not None:
self.branch = branch
def set_revision(self, revision):
"""Sets the current revision that is being tested"""
self.revision = revision
class Test:
"""A simple class to contain all information about tests"""
def __init__(self, name, command, ldopts, permutations, prof_command=None):
self.name = name
self.command = command
self.prof_command = prof_command
self.ldopts = ldopts.replace(' ', '').split(',') #Not tested yet
self.permutations = permutations
def parse_configfile(conffile, testdir, moses_repo, moses_prof_repo=None):
"""Parses the config file"""
command, ldopts, prof_command = '', '', None
permutations = []
fileopen = open(conffile, 'r')
for line in fileopen:
line = line.split('#')[0] # Discard comments
if line == '' or line == '\n':
continue # Discard lines with comments only and empty lines
opt, args = line.split(' ', 1) # Get arguments
if opt == 'Command:':
command = args.replace('\n', '')
if moses_prof is not None: # Get optional command for profiling
prof_command = moses_prof_repo + '/bin/' + command
command = moses_repo + '/bin/' + command
elif opt == 'LDPRE:':
ldopts = args.replace('\n', '')
elif opt == 'Variants:':
permutations = args.replace('\n', '').replace(' ', '').split(',')
else:
raise ValueError('Unrecognized option ' + opt)
#We use the testdir as the name.
testcase = Test(testdir, command, ldopts, permutations, prof_command)
fileopen.close()
return testcase
def parse_testconfig(conffile):
"""Parses the config file for the whole testsuite."""
repo_path, drop_caches, tests_dir, testlog_dir = '', '', '', ''
basebranch, baserev, repo_prof_path = '', '', None
fileopen = open(conffile, 'r')
for line in fileopen:
line = line.split('#')[0] # Discard comments
if line == '' or line == '\n':
continue # Discard lines with comments only and empty lines
opt, args = line.split(' ', 1) # Get arguments
if opt == 'MOSES_REPO_PATH:':
repo_path = args.replace('\n', '')
elif opt == 'DROP_CACHES_COMM:':
drop_caches = args.replace('\n', '')
elif opt == 'TEST_DIR:':
tests_dir = args.replace('\n', '')
elif opt == 'TEST_LOG_DIR:':
testlog_dir = args.replace('\n', '')
elif opt == 'BASEBRANCH:':
basebranch = args.replace('\n', '')
elif opt == 'BASEREV:':
baserev = args.replace('\n', '')
elif opt == 'MOSES_PROFILER_REPO:': # Optional
repo_prof_path = args.replace('\n', '')
else:
raise ValueError('Unrecognized option ' + opt)
config = Configuration(repo_path, drop_caches, tests_dir, testlog_dir,\
basebranch, baserev, repo_prof_path)
fileopen.close()
return config
def get_config():
"""Builds the config object with all necessary attributes"""
args = parse_cmd()
config = parse_testconfig(args.configfile)
config.additional_args(args.singletestdir, args.revision, args.branch)
revision = repoinit(config)
if config.repo_prof is not None:
repoinit(config, True)
config.set_revision(revision)
return config
def check_for_basever(testlogfile, basebranch):
"""Checks if the base revision is present in the testlogs"""
filetoopen = open(testlogfile, 'r')
for line in filetoopen:
templine = processLogLine(line)
if templine.branch == basebranch:
return True
return False
def split_time(filename):
"""Splits the output of the time function into seperate parts.
We will write time to file, because many programs output to
stderr which makes it difficult to get only the exact results we need."""
timefile = open(filename, 'r')
realtime = float(timefile.readline().replace('\n', '').split()[1])
usertime = float(timefile.readline().replace('\n', '').split()[1])
systime = float(timefile.readline().replace('\n', '').split()[1])
timefile.close()
return (realtime, usertime, systime)
def write_log(time_file, logname, config):
"""Writes to a logfile"""
log_write = open(config.testlogs + '/' + logname, 'a') # Open logfile
date_run = time.strftime("%d.%m.%Y %H:%M:%S") # Get the time of the test
realtime, usertime, systime = split_time(time_file) # Get the times in a nice form
# Append everything to a log file.
writestr = date_run + " " + config.revision + " Testname: " + logname +\
" RealTime: " + str(realtime) + " UserTime: " + str(usertime) +\
" SystemTime: " + str(systime) + " Branch: " + config.branch +'\n'
log_write.write(writestr)
log_write.close()
def write_gprof(command, name, variant, config):
"""Produces a gprof report from a gmon file"""
#Check if we have a directory for the profiling of this testcase:
output_dir = config.testlogs + '/' + name
if not os.path.exists(output_dir):
os.makedirs(output_dir)
outputfile = output_dir + '/' + time.strftime("%d.%m.%Y_%H:%M:%S") + '_' + name + '_' + variant
#Compile a gprof command and output the file in the directory we just created
gmon_path = os.getcwd() + '/gmon.out' # Path to the profiling file
executable_path = command.split(' ')[0] # Path to the moses binary
gprof_command = 'gprof ' + executable_path + ' ' + gmon_path + ' > ' + outputfile
subprocess.call([gprof_command], shell=True)
os.remove('gmon_path') # After we are done discard the gmon file
def execute_test(command, path, name, variant, config, profile=False):
"""Executes a testcase given a whole command, path to the test file output,
name of the test and variant tested. Config is the global configuration"""
subprocess.Popen([command], stdout=None, stderr=subprocess.PIPE, shell=True).communicate()
if not profile:
write_log(path, name + '_' + variant, config)
else: # Basically produce a gmon output
write_gprof(command, name, variant, config)
def execute_tests(testcase, cur_directory, config):
"""Executes timed tests based on the config file"""
#Several global commands related to the time wrapper
time_command = ' time -p -o /tmp/time_moses_tests '
time_path = '/tmp/time_moses_tests'
#Figure out the order of which tests must be executed.
#Change to the current test directory
os.chdir(config.tests + '/' + cur_directory)
#Clear caches
subprocess.call(['sync'], shell=True)
subprocess.call([config.drop_caches], shell=True)
#Perform vanilla test and if a cached test exists - as well
print(testcase.name)
if 'vanilla' in testcase.permutations:
#Create the command for executing moses
whole_command = time_command + testcase.command
#test normal and cached
execute_test(whole_command, time_path, testcase.name, 'vanilla', config)
if 'cached' in testcase.permutations:
execute_test(whole_command, time_path, testcase.name, 'vanilla_cached', config)
#Now perform LD_PRELOAD tests
if 'ldpre' in testcase.permutations:
for opt in testcase.ldopts:
#Clear caches
subprocess.call(['sync'], shell=True)
subprocess.call([config.drop_caches], shell=True)
#Create the command for executing moses:
whole_command = 'LD_PRELOAD=' + opt + time_command + testcase.command
variant = 'ldpre_' + opt
#test normal and cached
execute_test(whole_command, time_path, testcase.name, variant, config)
if 'cached' in testcase.permutations:
execute_test(whole_command, time_path, testcase.name, variant + '_cached', config)
#Perform profiling test. Mostly same as the above lines but necessary duplication.
#All actual code is inside execute_test so those lines shouldn't need modifying
if 'profile' in testcase.permutations:
subprocess.call(['sync'], shell=True) # Drop caches first
subprocess.call([config.drop_caches], shell=True)
if 'vanilla' in testcase.permutations:
whole_command = testcase.prof_command
execute_test(whole_command, time_path, testcase.name, 'profile', config, True)
if 'cached' in testcase.permutations:
execute_test(whole_command, time_path, testcase.name, 'profile_cached', config, True)
if 'ldpre' in testcase.permutations:
for opt in testcase.ldopts:
#Clear caches
subprocess.call(['sync'], shell=True)
subprocess.call([config.drop_caches], shell=True)
#Create the command for executing moses:
whole_command = 'LD_PRELOAD=' + opt + testcase.prof_command
variant = 'profile_ldpre_' + opt
#test normal and cached
execute_test(whole_command, time_path, testcase.name, variant, config, True)
if 'cached' in testcase.permutations:
execute_test(whole_command, time_path, testcase.name, variant + '_cached', config, True)
# Go through all the test directories and executes tests
if __name__ == '__main__':
CONFIG = get_config()
ALL_DIR = os.listdir(CONFIG.tests)
#We should first check if any of the tests is run for the first time.
#If some of them are run for the first time we should first get their
#time with the base version (usually the previous release)
FIRSTTIME = []
TESTLOGS = []
#Strip filenames of test underscores
for listline in os.listdir(CONFIG.testlogs):
listline = listline.replace('_vanilla', '')
listline = listline.replace('_cached', '')
listline = listline.replace('_ldpre', '')
TESTLOGS.append(listline)
for directory in ALL_DIR:
if directory not in TESTLOGS:
FIRSTTIME.append(directory)
#Sometimes even though we have the log files, we will need to rerun them
#Against a base version, because we require a different baseversion (for
#example when a new version of Moses is released.) Therefore we should
#Check if the version of Moses that we have as a base version is in all
#of the log files.
for logfile in os.listdir(CONFIG.testlogs):
logfile_name = CONFIG.testlogs + '/' + logfile
if not check_for_basever(logfile_name, CONFIG.basebranch):
logfile = logfile.replace('_vanilla', '')
logfile = logfile.replace('_cached', '')
logfile = logfile.replace('_ldpre', '')
FIRSTTIME.append(logfile)
FIRSTTIME = list(set(FIRSTTIME)) #Deduplicate
if FIRSTTIME != []:
#Create a new configuration for base version tests:
BASECONFIG = Configuration(CONFIG.repo, CONFIG.drop_caches,\
CONFIG.tests, CONFIG.testlogs, CONFIG.basebranch,\
CONFIG.baserev, CONFIG.repo_prof)
BASECONFIG.additional_args(None, CONFIG.baserev, CONFIG.basebranch)
#Set up the repository and get its revision:
REVISION = repoinit(BASECONFIG)
BASECONFIG.set_revision(REVISION)
#Build
os.chdir(BASECONFIG.repo)
subprocess.call(['./previous.sh'], shell=True)
#If profiler configuration exists also init it
if BASECONFIG.repo_prof is not None:
repoinit(BASECONFIG, True)
os.chdir(BASECONFIG.repo_prof)
subprocess.call(['./previous.sh'], shell=True)
#Perform tests
for directory in FIRSTTIME:
cur_testcase = parse_configfile(BASECONFIG.tests + '/' + directory +\
'/config', directory, BASECONFIG.repo)
execute_tests(cur_testcase, directory, BASECONFIG)
#Reset back the repository to the normal configuration
repoinit(CONFIG)
if BASECONFIG.repo_prof is not None:
repoinit(CONFIG, True)
#Builds moses
os.chdir(CONFIG.repo)
subprocess.call(['./previous.sh'], shell=True)
if CONFIG.repo_prof is not None:
os.chdir(CONFIG.repo_prof)
subprocess.call(['./previous.sh'], shell=True)
if CONFIG.singletest:
TESTCASE = parse_configfile(CONFIG.tests + '/' +\
CONFIG.singletest + '/config', CONFIG.singletest, CONFIG.repo)
execute_tests(TESTCASE, CONFIG.singletest, CONFIG)
else:
for directory in ALL_DIR:
cur_testcase = parse_configfile(CONFIG.tests + '/' + directory +\
'/config', directory, CONFIG.repo)
execute_tests(cur_testcase, directory, CONFIG)
|