12 from __future__
import print_function
19 from datetime
import datetime
20 from requests.auth
import HTTPBasicAuth
22 from ReporterMTTStage
import *
24 from requests.packages.urllib3.exceptions
import InsecureRequestWarning
25 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
48 ReporterMTTStage.__init__(self)
50 self.
options[
'realm'] = (
None,
"Database name")
51 self.
options[
'username'] = (
None,
"Username to be used for submitting data")
52 self.
options[
'password'] = (
None,
"Password for that username")
53 self.
options[
'pwfile'] = (
None,
"File where password can be found")
54 self.
options[
'platform'] = (
None,
"Name of the platform (cluster) upon which the tests were run")
55 self.
options[
'hostname'] = (
None,
"Name of the hosts involved in the tests (may be regular expression)")
56 self.
options[
'url'] = (
None,
"URL of the database server")
57 self.
options[
'debug_filename'] = (
None,
"Debug output file for server interaction information")
58 self.
options[
'keep_debug_files'] = (
False,
"Retain reporter debug output after execution")
59 self.
options[
'debug_server'] = (
False,
"Ask the server to return its debug output as well")
60 self.
options[
'email'] = (
None,
"Email to which errors are to be sent")
64 IPlugin.activate(self)
69 IPlugin.deactivate(self)
76 lines = testDef.printOptions(self.
options)
84 testDef.parseOptions(log, self.
options, keyvals, cmds)
88 if cmds[
'username']
is not None:
90 if cmds[
'password']
is not None or cmds[
'pwfile']
is not None:
92 if cmds[
'realm']
is not None:
94 if 0 < sanity
and sanity != 3:
96 log[
'stderr'] =
"MTTDatabase Reporter section",log[
'section'] +
": if password, username, or realm is specified, they all must be specified."
99 if cmds[
'pwfile']
is not None:
100 if os.path.exists(cmds[
'pwfile']):
101 f = open(cmds[
'pwfile'],
'r')
102 password = f.readline().strip()
106 log[
'stderr'] =
"Password file " + cmds[
'pwfile'] +
" does not exist"
108 elif cmds[
'password']
is not None:
109 password = cmds[
'password']
112 if cmds[
'password']
is not None:
113 password = cmds[
'password']
119 s = requests.Session()
120 url = cmds[
'url'] +
"/submit"
122 www_auth = HTTPBasicAuth(cmds[
'username'], password)
128 if client_serial < 0:
129 print(
"Error: Unable to get a client serial (rtn=%d)" % (client_serial))
132 headers[
'content-type'] =
'application/json'
136 profile = testDef.logger.getLog(
'Profile:Installed')
138 metadata[
'client_serial'] = client_serial
139 metadata[
'hostname'] =
"\n".join(profile[
'profile'][
'nodeName'])
140 metadata[
'http_username'] = cmds[
'username']
141 metadata[
'local_username'] = pwd.getpwuid(os.getuid()).pw_name
142 metadata[
'mtt_client_version'] =
'4.0a1'
143 metadata[
'platform_name'] = self.
_extract_param(testDef.logger,
'MTTDefaults',
'platform')
145 if trl
and trl.lower() !=
"false":
146 metadata[
'trial'] = 1
148 metadata[
'trial'] = 0
160 fullLog = testDef.logger.getLog(
None)
161 pp = pprint.PrettyPrinter(indent=4)
166 print(
"<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>>")
168 print(
"----------------- Section (%s) " % (lg[
'section']))
170 print(
"<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>>")
177 if re.match(
"TestRun", lg[
'section']):
189 print(
"----------------- Test Run (%s) " % (lg[
'section']))
191 pp = pprint.PrettyPrinter(indent=4)
196 logger.getLog(self.
_extract_param(logger, lg[
'section'],
'parent')),
200 if test_info
is None:
205 options = lg[
'options']
213 metadata[
'phase'] =
'Test Run'
219 common_data[
'test_build_id'] = test_info[
'test_build_id']
221 for trun
in (lg[
'testresults']
if 'testresults' in lg
else [lg]):
225 data[
'test_build_id'] = common_data[
'test_build_id']
228 data[
'launcher'] = options[
'command']
230 data[
'launcher'] =
None
232 if 'testresults' in lg:
233 data[
'test_name'] = trun[
'test'].split(
'/')[-1]
235 data[
'test_name'] = trun[
'options'][
'command'].split(
'/')[-1]
240 if 'testresults' in lg:
241 data[
'np'] = lg[
'np']
247 if 'testresults' in lg:
248 data[
'command'] = trun[
'cmd']
250 data[
'command'] = trun[
'options'][
'command']
253 data[
'start_timestamp'] = datetime.utcnow().strftime(
"%c")
256 status = trun[
'status']
260 data[
'result_message'] =
"Success"
261 data[
'test_result'] = 1
262 data[
'exit_value'] = 0
264 data[
'result_message'] =
"Failed"
265 data[
'test_result'] = 0
271 if type(lg[
'stderr'])
is list:
272 lgerr =
'\n'.join(lg[
'stderr'])
275 if '[Errno' in lgerr:
277 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
279 data[
'exit_value'] = -1
281 data[
'exit_value'] = -1
283 data[
'result_message'] =
"Failed"
284 data[
'test_result'] = -1
286 if type(lg[
'stderr'])
is list:
287 lgerr =
'\n'.join(lg[
'stderr'])
290 if '[Errno' in lgerr:
292 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
294 data[
'exit_value'] = -1
296 data[
'exit_value'] = -1
298 data[
'exit_value'] = -1
319 data[
'description'] = self.
_extract_param(logger,
'MTTDefaults',
'description')
322 for lgentry
in logger.getLog(
None):
323 if 'environ' in lgentry:
324 environment.update(lgentry[
'environ'])
325 data[
'environment'] =
"\n".join([str(k) +
"=" + str(v)
for k,v
in environment.items()])
330 for lgentry
in logger.getLog(
None):
331 if 'bios' in lgentry:
332 bios.update(lgentry[
'bios'])
334 data[
'bios_nodelist'] = bios[
'nodelist']
335 data[
'bios_params'] = bios[
'params']
336 data[
'bios_values'] = bios[
'values']
342 for lgentry
in logger.getLog(
None):
343 if 'firmware' in lgentry:
344 firmware.update(lgentry[
'firmware'])
346 data[
'flashupdt_cfg'] = firmware[
'flashupdt_cfg']
347 data[
'firmware_nodelist'] = firmware[
'nodelist']
353 for lgentry
in logger.getLog(
None):
354 if 'provisioning' in lgentry:
355 provisioning.update(lgentry[
'provisioning'])
357 data[
'targets'] = provisioning[
'target']
358 data[
'image'] = provisioning[
'image']
359 data[
'controllers'] = provisioning[
'controller']
360 data[
'bootstrap'] = provisioning[
'bootstrap']
366 for lgentry
in logger.getLog(
None):
367 if 'harasser' in lgentry:
368 harasser.update(lgentry[
'harasser'])
370 data[
'harasser_seed'] = harasser[
'seed']
371 data[
'inject_script'] = harasser[
'inject_script']
372 data[
'cleanup_script'] = harasser[
'cleanup_script']
373 data[
'check_script'] = harasser[
'check_script']
379 if options[
'merge_stdout_stderr']:
380 data[
'merge_stdout_stderr'] = 1
382 data[
'merge_stdout_stderr'] = 0
384 data[
'merge_stdout_stderr'] =
None
387 data[
'result_stdout'] =
'\n'.join(trun[
'stdout']
if trun[
'stdout']
is not None else "")
389 data[
'result_stdout'] =
None
392 if type(trun[
'stderr'])
is list:
393 data[
'result_stderr'] =
'\n'.join(trun[
'stderr']
if trun[
'stderr']
is not None else "")
395 data[
'result_stderr'] = (trun[
'stderr']
if trun[
'stderr']
is not None else "")
397 data[
'result_stderr'] =
None
403 payload[
'metadata'] = metadata
404 payload[
'data'] = [data]
409 if data[
'status']
is not 0:
415 print(
"----------------- Test Build (%s) " % (lg[
'section']))
417 pp = pprint.PrettyPrinter(indent=4)
426 if install_info
is None:
431 options = lg[
'options']
440 metadata[
'phase'] =
'Test Build'
443 data[
'mpi_install_id'] = install_info[
'mpi_install_id']
446 data[
'compiler_name'] = lg[
'compiler'][
'compiler']
448 data[
'compiler_version'] = lg[
'compiler'][
'version']
450 full_log = logger.getLog(
None)
451 for entry
in full_log:
452 if 'compiler' in entry:
453 data[
'compiler_name'] = entry[
'compiler'][
'compiler']
455 data[
'compiler_version'] = entry[
'compiler'][
'version']
458 data[
'compiler_name'] =
None
459 data[
'compiler_version'] =
None
461 data[
'suite_name'] = lg[
'section']
464 data[
'start_timestamp'] = datetime.utcnow().strftime(
"%c")
467 status = lg[
'status']
471 data[
'result_message'] =
"Success"
472 data[
'test_result'] = 1
473 data[
'exit_value'] = 0
475 data[
'result_message'] =
"Failed"
476 data[
'test_result'] = 0
482 if type(lg[
'stderr'])
is list:
483 lgerr =
'\n'.join(lg[
'stderr'])
486 if '[Errno' in lgerr:
488 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
490 data[
'exit_value'] = -1
492 data[
'exit_value'] = -1
494 data[
'exit_value'] = -1
496 data[
'result_message'] =
"Failed"
497 data[
'test_result'] = -1
503 if type(lg[
'stderr'])
is list:
504 lgerr =
'\n'.join(lg[
'stderr'])
507 if '[Errno' in lgerr:
509 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
511 data[
'exit_value'] = -1
513 data[
'exit_value'] = -1
515 data[
'exit_value'] = -1
522 data[
'description'] = self.
_extract_param(logger,
'MTTDefaults',
'description')
525 for lgentry
in logger.getLog(
None):
526 if 'environ' in lgentry:
527 environment.update(lgentry[
'environ'])
528 data[
'environment'] =
"\n".join([str(k) +
"=" + str(v)
for k,v
in environment.items()])
531 if options[
'merge_stdout_stderr']:
532 data[
'merge_stdout_stderr'] = 1
534 data[
'merge_stdout_stderr'] = 0
536 data[
'merge_stdout_stderr'] =
None
539 data[
'result_stdout'] =
'\n'.join(lg[
'stdout'])
541 data[
'result_stdout'] =
None
544 if type(lg[
'stderr'])
is list:
545 data[
'result_stderr'] =
'\n'.join(lg[
'stderr'])
547 data[
'result_stderr'] = lg[
'stderr']
549 data[
'result_stderr'] =
None
555 payload[
'metadata'] = metadata
556 payload[
'data'] = [data]
561 if data[
'status']
is not 0:
565 return self.
_merge_dict( {
'test_build_id':data[
'ids'][0][
'test_build_id']},
570 print(
"----------------- MPI Install (%s) " % (lg[
'section']))
571 pp = pprint.PrettyPrinter(indent=4)
587 profile = logger.getLog(
'Profile:Installed')[
'profile']
589 print(
"Error: Failed to get 'profile'")
597 metadata[
'phase'] =
'MPI Install'
600 data[
'platform_hardware'] =
"\n".join(profile[
'machineName'])
602 data[
'platform_hardware'] =
None
605 data[
'platform_type'] =
"\n".join(profile[
'processorType'])
607 data[
'platform_type'] =
None
610 data[
'os_name'] =
"\n".join(profile[
'kernelName'])
612 data[
'os_name'] =
None
615 data[
'os_version'] =
"\n".join(profile[
'kernelRelease'])
617 data[
'os_version'] =
None
620 data[
'compiler_name'] = lg[
'compiler'][
'compiler']
622 data[
'compiler_version'] = lg[
'compiler'][
'version']
624 full_log = logger.getLog(
None)
625 for entry
in full_log:
626 if 'compiler' in entry:
627 data[
'compiler_name'] = entry[
'compiler'][
'compiler']
629 data[
'compiler_version'] = entry[
'compiler'][
'version']
632 data[
'compiler_name'] =
None
633 data[
'compiler_version'] =
None
636 data[
'mpi_name'] = lg[
'mpi_info'][
'name']
637 data[
'mpi_version'] = lg[
'mpi_info'][
'version']
639 full_log = logger.getLog(
None)
640 for entry
in full_log:
641 if 'mpi_info' in entry:
642 data[
'mpi_name'] = entry[
'mpi_info'][
'name']
643 data[
'mpi_version'] = entry[
'mpi_info'][
'version']
646 data[
'mpi_name'] =
None
647 data[
'mpi_version'] =
None
650 data[
'configure_arguments'] = logger.getLog(lg[
'middleware'])[
'configure_options']
651 except (KeyError, TypeError):
652 data[
'configure_arguments'] =
None
655 data[
'start_timestamp'] = datetime.utcnow().strftime(
"%c")
658 status = lg[
'status']
662 data[
'result_message'] =
"Success"
663 data[
'test_result'] = 1
664 data[
'exit_value'] = 0
666 data[
'result_message'] =
"Failed"
667 data[
'test_result'] = 0
673 if type(lg[
'stderr'])
is list:
674 lgerr =
'\n'.join(lg[
'stderr'])
677 if '[Errno' in lgerr:
679 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
681 data[
'exit_value'] = -1
683 data[
'exit_value'] = -1
685 data[
'exit_value'] = -1
687 data[
'result_message'] =
"Failed"
688 data[
'test_result'] = -1
694 if type(lg[
'stderr'])
is list:
695 lgerr =
'\n'.join(lg[
'stderr'])
698 if '[Errno' in lgerr:
700 data[
'exit_value'] = int(lgerr.split(
"[Errno ")[1].split(
"]")[0])
702 data[
'exit_value'] = -1
704 data[
'exit_value'] = -1
706 data[
'exit_value'] = -1
717 data[
'description'] = self.
_extract_param(logger,
'MTTDefaults',
'description')
720 for lgentry
in logger.getLog(
None):
721 if 'environ' in lgentry:
722 environment.update(lgentry[
'environ'])
723 data[
'environment'] =
"\n".join([str(k) +
"=" + str(v)
for k,v
in environment.items()])
726 if options
is not None and options[
'merge_stdout_stderr']:
727 data[
'merge_stdout_stderr'] = 1
729 data[
'merge_stdout_stderr'] = 0
731 data[
'merge_stdout_stderr'] =
None
734 data[
'result_stdout'] =
'\n'.join(lg[
'stdout'])
736 data[
'result_stdout'] =
None
739 if type(lg[
'stderr'])
is list:
740 data[
'result_stderr'] =
'\n'.join(lg[
'stderr'])
742 data[
'result_stderr'] = lg[
'stderr']
744 data[
'result_stderr'] =
None
750 payload[
'metadata'] = metadata
751 payload[
'data'] = [data]
756 if data[
'status']
is not 0:
760 return {
'mpi_install_id':data[
'ids'][0][
'mpi_install_id']}
764 headers[
'content-type'] =
'application/json'
766 print(
"<<<<<<<---------------- Payload (Start) -------------------------->>>>>>")
767 print(json.dumps(payload, sort_keys=
True, indent=4, separators=(
',',
': ')))
768 print(
"<<<<<<<---------------- Payload (End ) -------------------------->>>>>>")
771 data=json.dumps(payload),
776 print(
"<<<<<<<---------------- Response -------------------------->>>>>>")
777 print(
"Result: %d: %s" % (r.status_code, r.headers[
'content-type']))
780 print(
"<<<<<<<---------------- Raw Output (Start) ---------------->>>>>>")
782 print(
"<<<<<<<---------------- Raw Output (End ) ---------------->>>>>>")
784 if r.status_code != 200:
790 found = logger.getLog(section)
792 print(
"_extract_param: Section (%s) Not Found! [param=%s]" % (section, parameter))
796 params = found[
'parameters']
798 print(
"_extract_param: Section (%s) did not contain a parameters entry! [param=%s]" % (section, parameter))
801 if p[0] == parameter:
805 url = url +
"/serial"
808 headers[
'content-type'] =
'application/json'
811 payload[
'serial'] =
'serial'
817 if data[
'status']
is not 0:
820 return data[
'client_serial']