11 from __future__
import print_function
13 from LauncherMTTTool
import *
45 LauncherMTTTool.__init__(self)
47 self.
options[
'hostfile'] = (
None,
"The hostfile for ALPS to use")
48 self.
options[
'command'] = (
"aprun",
"Command for executing the application")
49 self.
options[
'np'] = (
None,
"Number of processes to run")
50 self.
options[
'options'] = (
None,
"Comma-delimited sets of command line options that shall be used on each test")
51 self.
options[
'skipped'] = (
"77",
"Exit status of a test that declares it was skipped")
52 self.
options[
'merge_stdout_stderr'] = (
False,
"Merge stdout and stderr into one output stream")
53 self.
options[
'stdout_save_lines'] = (-1,
"Number of lines of stdout to save")
54 self.
options[
'stderr_save_lines'] = (-1,
"Number of lines of stderr to save")
55 self.
options[
'test_dir'] = (
None,
"Names of directories to be scanned for tests")
56 self.
options[
'fail_tests'] = (
None,
"Names of tests that are expected to fail")
57 self.
options[
'fail_returncodes'] = (
None,
"Expected return code of tests expected to fail")
58 self.
options[
'fail_timeout'] = (
None,
"Maximum execution time for tests expected to fail")
59 self.
options[
'skip_tests'] = (
None,
"Names of tests to be skipped")
60 self.
options[
'max_num_tests'] = (
None,
"Maximum number of tests to run")
61 self.
options[
'modules'] = (
None,
"Modules to load")
62 self.
options[
'modules_unload'] = (
None,
"Modules to unload")
63 self.
options[
'test_list'] = (
None,
"List of tests to run, default is all")
64 self.
options[
'allocate_cmd'] = (
None,
"Command to use for allocating nodes from the resource manager")
65 self.
options[
'deallocate_cmd'] = (
None,
"Command to use for deallocating nodes from the resource manager")
75 IPlugin.activate(self)
80 IPlugin.deactivate(self)
82 deallocate_cmdargs = shlex.split(self.
cmds[
'deallocate_cmd'])
83 _status,_stdout,_stderr,_time = self.testDef.execmd.execute(self.
cmds, deallocate_cmdargs, self.
testDef)
91 lines = testDef.printOptions(self.
options)
102 testDef.logger.verbose_print(
"ALPS Launcher")
106 if log[
'section']
is not None:
107 if "Default" in log[
'section']:
111 testDef.parseOptions(log, self.
options, keyvals, myopts)
113 keys = list(self.options.keys())
114 optkeys = list(myopts.keys())
115 for optkey
in optkeys:
127 log[
'stderr'] =
"Section not specified"
132 parent = keyvals[
'parent']
133 if parent
is not None:
136 bldlog = testDef.logger.getLog(parent)
138 location = bldlog[
'location']
143 log[
'stderr'] =
"Location of built tests was not provided"
146 if bldlog[
'parameters']
is not None:
148 for md
in bldlog[
'parameters']:
149 if "modules_unload" == md[0]:
151 if keyvals[
'modules_unload']
is not None:
153 mods = md[1].split(
',')
154 newmods = keyvals[
'modules_unload'].split(
',')
157 keyvals[
'modules_unload'] =
','.join(mods)
159 keyvals[
'modules_unload'] = md[1]
162 for md
in bldlog[
'parameters']:
163 if "modules" == md[0]:
165 if keyvals[
'modules']
is not None:
167 mods = md[1].split(
',')
168 newmods = keyvals[
'modules'].split(
',')
171 keyvals[
'modules'] =
','.join(mods)
173 keyvals[
'modules'] = md[1]
179 midlog = testDef.logger.getLog(bldlog[
'middleware'])
180 if midlog
is not None:
183 if midlog[
'location']
is not None:
186 oldbinpath = os.environ[
'PATH']
187 pieces = oldbinpath.split(
':')
191 bindir = os.path.join(midlog[
'location'],
"bin")
192 pieces.insert(0, bindir)
193 newpath =
":".join(pieces)
194 os.environ[
'PATH'] = newpath
197 oldldlibpath = os.environ[
'LD_LIBRARY_PATH']
198 pieces = oldldlibpath.split(
':')
202 bindir = os.path.join(midlog[
'location'],
"lib")
203 pieces.insert(0, bindir)
204 newpath =
":".join(pieces)
205 os.environ[
'LD_LIBRARY_PATH'] = newpath
213 if midlog[
'parameters']
is not None:
215 for md
in midlog[
'parameters']:
216 if "modules_unload" == md[0]:
218 if keyvals[
'modules_unload']
is not None:
220 mods = md[1].split(
',')
221 newmods = keyvals[
'modules_unload'].split(
',')
224 keyvals[
'modules_unload'] =
','.join(mods)
226 keyvals[
'modules_unload'] = md[1]
229 for md
in midlog[
'parameters']:
230 if "modules" == md[0]:
232 if keyvals[
'modules']
is not None:
234 mods = md[1].split(
',')
235 newmods = keyvals[
'modules'].split(
',')
238 keyvals[
'modules'] =
','.join(mods)
240 keyvals[
'modules'] = md[1]
248 log[
'stderr'] =
"Parent test build stage was not provided"
252 testDef.parseOptions(log, self.
options, keyvals, cmds)
256 if cmds[
'command'] !=
"aprun":
258 log[
'stderr'] =
"Command for ALPS plugin must be aprun. It is currently '%s'" % (cmds[
'command']
if cmds[
'command']
is not None else "None")
270 if cmds[
'test_list']
is None:
272 if cmds[
'test_dir']
is not None:
274 dirs = cmds[
'test_dir'].split()
278 dr = dr.replace(
'\"',
'')
279 dr = dr.replace(
',',
'')
280 for dirName, subdirList, fileList
in os.walk(dr):
281 for fname
in fileList:
283 filename = os.path.abspath(os.path.join(dirName,fname))
284 if os.path.isfile(filename)
and os.access(filename, os.X_OK):
286 tests.append(filename)
290 for dirName, subdirList, fileList
in os.walk(
"."):
291 for fname
in fileList:
293 filename = os.path.abspath(os.path.join(dirName,fname))
294 if os.path.isfile(filename)
and os.access(filename, os.X_OK):
296 tests.append(filename)
300 for dirName, subdirList, fileList
in os.walk(
"."):
301 for fname
in fileList:
303 filename = os.path.abspath(os.path.join(dirName,fname))
304 if os.path.isfile(filename)
and os.access(filename, os.X_OK):
306 tests.append(filename)
309 if cmds[
'test_dir']
is not None:
310 dirs = cmds[
'test_dir'].split()
315 dr = dr.replace(
'\"',
'')
316 dr = dr.replace(
',',
'')
317 for dirName, subdirList, fileList
in os.walk(dr):
318 for fname
in cmds[
'test_list'].split(
","):
319 fname = fname.strip()
320 if fname
not in fileList:
322 filename = os.path.abspath(os.path.join(dirName,fname))
323 if os.path.isfile(filename)
and os.access(filename, os.X_OK):
324 tests.append(filename)
329 log[
'stderr'] =
"No tests found"
333 skipStatus = int(cmds[
'skipped'])
335 cmdargs = [cmds[
'command']]
336 if cmds[
'options']
is not None:
337 for op
in cmds[
'options'].split():
339 cmdargs.append(cmds[
'options'])
340 if cmds[
'np']
is not None:
342 cmdargs.append(cmds[
'np'])
343 if cmds[
'hostfile']
is not None:
344 cmdargs.append(
"--node-list-file")
345 cmdargs.append(cmds[
'hostfile'])
347 log[
'testresults'] = []
354 if cmds[
'max_num_tests']
is not None:
355 maxTests = int(cmds[
'max_num_tests'])
360 usedModuleUnload =
False
362 if cmds[
'modules_unload']
is not None:
363 status,stdout,stderr = testDef.modcmd.unloadModules(cmds[
'modules_unload'], testDef)
365 log[
'status'] = status
366 log[
'stderr'] = stderr
369 usedModuleUnload =
True
376 if cmds[
'modules']
is not None:
377 status,stdout,stderr = testDef.modcmd.loadModules(cmds[
'modules'], testDef)
379 log[
'status'] = status
380 log[
'stderr'] = stderr
388 fail_tests = cmds[
'fail_tests']
389 if fail_tests
is not None:
390 fail_tests = [t.strip()
for t
in fail_tests.split(
",")]
393 for i,t
in enumerate(fail_tests):
395 if t2.split(
"/")[-1] == t:
397 fail_returncodes = cmds[
'fail_returncodes']
398 if fail_returncodes
is not None:
399 fail_returncodes = [int(t.strip())
for t
in fail_returncodes.split(
",")]
401 if fail_tests
is None:
402 expected_returncodes = {test:0
for test
in tests}
404 if fail_returncodes
is None:
405 expected_returncodes = {test:(
None if test
in fail_tests
else 0)
for test
in tests}
407 fail_returncodes = {test:rtncode
for test,rtncode
in zip(fail_tests,fail_returncodes)}
408 expected_returncodes = {test:(fail_returncodes[test]
if test
in fail_returncodes
else 0)
for test
in tests}
412 if cmds[
'allocate_cmd']
is not None and cmds[
'deallocate_cmd']
is not None:
414 allocate_cmdargs = shlex.split(cmds[
'allocate_cmd'])
415 _status,_stdout,_stderr,_time = testDef.execmd.execute(cmds, allocate_cmdargs, testDef)
417 log[
'status'] = _status
418 log[
'stderr'] = _stderr
425 if cmds[
'skip_tests']
is not None and test.split(
'/')[-1]
in [st.strip()
for st
in cmds[
'skip_tests'].split()]:
428 if numTests == maxTests:
431 testLog = {
'test':test}
433 testLog[
'cmd'] =
" ".join(cmdargs)
435 harass_exec_ids = testDef.harasser.start(testDef)
437 harass_check = testDef.harasser.check(harass_exec_ids, testDef)
438 if harass_check
is not None:
439 testLog[
'stderr'] =
'Not all harasser scripts started. These failed to start: ' \
440 +
','.join([h_info[1][
'start_script']
for h_info
in harass_check[0]])
441 testLog[
'time'] = sum([r_info[3]
for r_info
in harass_check[1]])
442 testLog[
'status'] = 1
444 finalError = testLog[
'stderr']
445 numFail = numFail + 1
446 testDef.harasser.stop(harass_exec_ids, testDef)
449 status,stdout,stderr,time = testDef.execmd.execute(cmds, cmdargs, testDef)
451 testDef.harasser.stop(harass_exec_ids, testDef)
453 if 0 != status
and skipStatus != status
and 0 == finalStatus:
454 if expected_returncodes[test] == 0:
459 if ((expected_returncodes[test]
is None and 0 == status)
or (expected_returncodes[test]
is not None and expected_returncodes[test] != status))
and skipStatus != status
and 0 == finalStatus:
460 numPass = numPass + 1
461 elif skipStatus == status:
462 numSkip = numSkip + 1
464 numFail = numFail + 1
465 if expected_returncodes[test] == 0:
466 testLog[
'status'] = status
468 if status == expected_returncodes[test]:
469 testLog[
'status'] = 0
471 testLog[
'status'] = 1
472 testLog[
'stdout'] = stdout
473 testLog[
'stderr'] = stderr
474 testLog[
'time'] = time
475 log[
'testresults'].append(testLog)
476 cmdargs = cmdargs[:-1]
477 numTests = numTests + 1
478 if numTests == maxTests:
482 if cmds[
'allocate_cmd']
is not None and cmds[
'deallocate_cmd']
is not None and self.
allocated:
483 deallocate_cmdargs = shlex.split(cmds[
'deallocate_cmd'])
484 _status,_stdout,_stderr,_time = testDef.execmd.execute(cmds, deallocate_cmdargs, testDef)
486 log[
'status'] = _status
487 log[
'stderr'] = _stderr
492 log[
'status'] = finalStatus
493 log[
'stderr'] = finalError
494 log[
'numTests'] = numTests
495 log[
'numPass'] = numPass
496 log[
'numSkip'] = numSkip
497 log[
'numFail'] = numFail
500 if 'np' not in cmds
or cmds[
'np']
is None:
501 if '-n ' in cmds[
'options']:
502 log[
'np'] = str(cmds[
'options'].split(
'-n ')[1].split(
' ')[0])
503 elif '--ntasks=' in cmds[
'options']:
504 log[
'np'] = str(cmds[
'options'].split(
'--ntasks=')[1].split(
' ')[0])
505 elif '-N ' in cmds[
'options']:
506 log[
'np'] = str(cmds[
'options'].split(
'-N ')[1].split(
' ')[0])
507 elif '--nodes=' in cmds[
'options']:
508 log[
'np'] = str(cmds[
'options'].split(
'--nodes=')[1].split(
' ')[0])
512 log[
'np'] = cmds[
'np']
516 status,stdout,stderr = testDef.modcmd.unloadModules(cmds[
'modules'], testDef)
518 log[
'status'] = status
519 log[
'stderr'] = stderr
523 status,stdout,stderr = testDef.modcmd.loadModules(cmds[
'modules_unload'], testDef)
525 log[
'status'] = status
526 log[
'stderr'] = stderr
532 os.environ[
'PATH'] = oldbinpath
533 os.environ[
'LD_LIBRARY_PATH'] = oldldlibpath