diff --git a/lnt/lnttool/import_report.py b/lnt/lnttool/import_report.py index de6b98ac..7f4bd42e 100644 --- a/lnt/lnttool/import_report.py +++ b/lnt/lnttool/import_report.py @@ -28,24 +28,29 @@ def action_importreport(input, output, suite, order, machine): import lnt.testing import os - machine_info = {} - run_info = {'tag': suite} - run_info['run_order'] = order - machine = lnt.testing.Machine(machine, - machine_info) + machine = lnt.testing.Machine(machine) + ctime = os.path.getctime(input.name) mtime = os.path.getmtime(input.name) + run = lnt.testing.Run(start_time=ctime, end_time=mtime, + info={'llvm_project_revision': order}) - run = lnt.testing.Run(ctime, mtime, run_info) - report = lnt.testing.Report(machine=machine, run=run, tests=[]) - + tests = {} # name => lnt.testing.Test for line in input.readlines(): key, val = line.split() - metric = key.split(".")[1] + (testname, metric) = key.split(".") metric_type = float if metric not in ("hash", "profile") else str - test = lnt.testing.TestSamples(suite + "." + key, [val], - conv_f=metric_type) - report.tests.extend([test]) + if testname not in tests: + tests[testname] = lnt.testing.Test(testname, []) + test = tests[testname] + + samples = next((s for s in test.samples if s.metric == metric), None) + if samples is None: + test.samples.append(lnt.testing.MetricSamples(metric, [])) + samples = test.samples[-1] + + samples.add_samples([val], conv_f=metric_type) + report = lnt.testing.Report(machine=machine, run=run, tests=list(tests.values())) output.write(report.render()) diff --git a/lnt/testing/__init__.py b/lnt/testing/__init__.py index 5b9f68ab..28374253 100644 --- a/lnt/testing/__init__.py +++ b/lnt/testing/__init__.py @@ -32,66 +32,41 @@ class Report: In the LNT test model, every test run should define exactly one machine and run, and any number of test samples. """ - def __init__(self, machine, run, tests, report_version=1): - """Construct a LNT report file format in the given format version.""" + def __init__(self, machine, run, tests): + """Construct a LNT report file.""" self.machine = machine self.run = run self.tests = list(tests) - self.report_version = report_version - self.check() + self._check() - def check(self): + def _check(self): """Check that object members are adequate to generate an LNT - json report file of the version specified at construction when - rendering that instance. + json report file. """ - # Check requested report version is supported by this library - assert self.report_version <= 2, "Only v2 or older LNT report format supported." - assert isinstance(self.machine, Machine), "Unexpected type for machine." - assert ( - self.machine.report_version == self.report_version - ), "Mismatch between machine and report version." - assert isinstance(self.run, Run), "Unexpected type for run." - assert ( - self.run.report_version == self.report_version - ), "Mismatch between run and report version." - for t in self.tests: - if self.report_version == 2: - assert isinstance(t, Test), "Unexpected type for test" - assert ( - t.report_version == self.report_version - ), "Mismatch between test and report version." - else: - assert isinstance(t, TestSamples), "Unexpected type for test samples." + assert isinstance(t, Test), "Unexpected type for test" def update_report(self, new_tests_samples, end_time=None): """Add extra samples to this report, and update the end time of the run. """ - self.check() + self._check() self.tests.extend(new_tests_samples) self.run.update_endtime(end_time) - self.check() + self._check() def render(self, indent=4): - """Return a LNT json report file format of the version specified - at construction as a string, where each object is indented by - indent spaces compared to its parent. + """Return a LNT json report file as a string, where each object is + indented by spaces compared to its parent. """ - if self.report_version == 2: - return json.dumps({'format_version': str(self.report_version), - 'machine': self.machine.render(), - 'run': self.run.render(), - 'tests': [t.render() for t in self.tests]}, - sort_keys=True, indent=indent) - else: - return json.dumps({'Machine': self.machine.render(), - 'Run': self.run.render(), - 'Tests': [t.render() for t in self.tests]}, - sort_keys=True, indent=indent) + return json.dumps({ + 'format_version': '2', + 'machine': self.machine.render(), + 'run': self.run.render(), + 'tests': [t.render() for t in self.tests]}, + sort_keys=True, indent=indent) class Machine: @@ -104,44 +79,25 @@ class Machine: Machines entries in the database are uniqued by their name and the entire contents of the info dictionary. """ - def __init__(self, name, info={}, report_version=1): + def __init__(self, name, info={}): self.name = str(name) self.info = dict((str(key), str(value)) for key, value in info.items()) - self.report_version = report_version - self.check() - - def check(self): - """Check object members are adequate to generate an LNT json - report file of the version specified at construction when - rendering that instance. - """ - # Check requested version is supported by this library - assert ( - self.report_version <= 2 - ), "Only v2 or older supported for LNT report format Machine objects." def render(self): """Return info from this instance in a dictionary that respects - the LNT report format in the version specified at construction - when printed as json. + the LNT JSON report format. """ - if self.report_version == 2: - d = dict(self.info) - d['Name'] = self.name - return d - else: - return {'Name': self.name, - 'Info': self.info} + d = dict(self.info) + d['name'] = self.name + return d class Run: """Information on the particular test run. At least one parameter must be supplied and is used as ordering - among several runs. When generating a report in format 1 or earlier, - both start_time and end_time are used for that effect and the - current date is used if their value is None. + among several runs. As with Machine, the info dictionary can be used to describe additional information on the run. This dictionary should be used to @@ -151,12 +107,7 @@ class Run: which could be useful in analysis, for example the current machine load. """ - def __init__(self, start_time=None, end_time=None, info={}, report_version=1): - if report_version <= 1: - if start_time is None: - start_time = datetime.datetime.utcnow() - if end_time is None: - end_time = datetime.datetime.utcnow() + def __init__(self, start_time=None, end_time=None, info={}): self.start_time = normalize_time(start_time) if start_time is not None else None self.end_time = normalize_time(end_time) if end_time is not None else None self.info = dict() @@ -165,59 +116,24 @@ def __init__(self, start_time=None, end_time=None, info={}, report_version=1): key = str(key) value = str(value) self.info[key] = value - self.report_version = report_version - if self.report_version <= 1: - if 'tag' not in self.info: - raise ValueError("Missing 'tag' entry in 'info' dictionary") - if 'run_order' not in self.info: - raise ValueError("Missing 'run_order' entry in 'info' dictionary") - else: - if 'llvm_project_revision' not in self.info: - raise ValueError("Missing 'llvm_project_revision' entry in 'info' dictionary") - if '__report_version__' in info: - raise ValueError("'__report_version__' key is reserved") - if report_version == 1: - self.info['__report_version__'] = '1' - self.check() - def check(self): - """Check object members are adequate to generate an LNT json - report file of the version specified at construction when - rendering that instance. - """ - # Check requested version is supported by this library - assert ( - self.report_version <= 2 - ), "Only v2 or older supported for LNT report format Run objects." - if self.start_time is None and self.end_time is None and not bool(self.info): - raise ValueError("No data defined in this Run") + if 'llvm_project_revision' not in self.info: + raise ValueError("Missing 'llvm_project_revision' entry in 'info' dictionary") def update_endtime(self, end_time=None): """Update the end time of this run.""" - if self.report_version <= 1 and end_time is None: - end_time = datetime.datetime.utcnow() self.end_time = normalize_time(end_time) if end_time else None - self.check() def render(self): """Return info from this instance in a dictionary that respects - the LNT report format in the version specified at construction - when printed as json. + the LNT JSON report format. """ - if self.report_version == 2: - d = dict(self.info) - if self.start_time is not None: - d['start_time'] = self.start_time - if self.end_time is not None: - d['end_time'] = self.end_time - return d - else: - info = dict(self.info) - if self.report_version == 1: - info['__report_version__'] = '1' - return {'Start Time': self.start_time, - 'End Time': self.end_time, - 'Info': info} + d = dict(self.info) + if self.start_time is not None: + d['start_time'] = self.start_time + if self.end_time is not None: + d['end_time'] = self.end_time + return d class Test: @@ -225,8 +141,7 @@ class Test: samples. The server automatically creates test database objects whenever a - new test name is seen. Test should be used to generate report in - version 2 or later of LNT JSON report file format. + new test name is seen. Test names are intended to be a persistent, recognizable identifier for what is being executed. Currently, most formats use some form of @@ -242,10 +157,10 @@ class Test: for example, the compile flags the test was built with, or the runtime parameters that were used. As a general rule, if two test samples are meaningfully and directly comparable, then they should - have the same test name but different info paramaters. + have the same test name but different info parameters. """ - def __init__(self, name, samples, info={}, report_version=2): + def __init__(self, name, samples, info={}): self.name = name self.samples = samples self.info = dict() @@ -254,33 +169,22 @@ def __init__(self, name, samples, info={}, report_version=2): key = str(key) value = str(value) self.info[key] = value - self.report_version = report_version self.check() def check(self): """Check object members are adequate to generate an LNT json - report file of the version specified at construction when - rendering that instance. + report file. """ - # Check requested version is supported by this library and is - # valid for this object. - assert ( - self.report_version == 2 - ), "Only v2 supported for LNT report format Test objects." for s in self.samples: assert isinstance(s, MetricSamples), "Unexpected type for metric sample." - assert ( - s.report_version == self.report_version - ), "Mismatch between test and metric samples." def render(self): """Return info from this instance in a dictionary that respects - the LNT report format in the version specified at construction - when printed as json. + the LNT JSON report format. """ d = dict(self.info) d.update([s.render().popitem() for s in self.samples]) - d['Name'] = self.name + d['name'] = self.name return d @@ -309,7 +213,7 @@ class TestSamples: for example, the compile flags the test was built with, or the runtime parameters that were used. As a general rule, if two test samples are meaningfully and directly comparable, then they should - have the same test name but different info paramaters. + have the same test name but different info parameters. The report may include an arbitrary number of samples for each test for situations where the same test is run multiple times to gather @@ -347,27 +251,11 @@ class MetricSamples: An arbitrary number of samples for a given metric is allowed for situations where the same metric is obtained several time for a given test to gather statistical data. - - MetricSamples should be used to generate report in version 2 or - later of LNT JSON report file format. """ - def __init__(self, metric, data, conv_f=float, report_version=2): + def __init__(self, metric, data, conv_f=float): self.metric = str(metric) self.data = list(map(conv_f, data)) - self.report_version = report_version - self.check() - - def check(self): - """Check object members are adequate to generate an LNT json - report file of the version specified at construction when - rendering that instance. - """ - # Check requested version is supported by this library and is - # valid for this object. - assert ( - self.report_version == 2 - ), "Only v2 supported for LNT report format MetricSamples objects." def add_samples(self, new_samples, conv_f=float): """Add samples for this metric, converted to float by calling @@ -389,8 +277,10 @@ def render(self): # We record information on the report "version" to allow the server to support # some level of auto-upgrading data from submissions of older reports. # -# We recorder the report version as a reserved key in the run information -# (primarily so that it can be accessed post-import on the server). +# We record the report version as a reserved key in the run information. When +# importing data, we detect the version of the report using the version number +# and we normalize it to the latest format so that the rest of the code only +# has to deal with the latest version at all times. # # Version 0 -- : initial (and unversioned). # diff --git a/lnt/tests/nt.py b/lnt/tests/nt.py index f31a3b49..82b324dd 100644 --- a/lnt/tests/nt.py +++ b/lnt/tests/nt.py @@ -729,7 +729,7 @@ def execute_nt_tests(test_log, make_variables, basedir, config): # Keep a mapping of mangled test names, to the original names in the # test-suite. TEST_TO_NAME = {} -KNOWN_SAMPLE_KEYS = ('compile', 'exec', 'hash', +KNOWN_SAMPLE_KEYS = ('compile_time', 'execution_time', 'hash', 'gcc.compile', 'bc.compile', 'llc.compile', 'llc-beta.compile', 'jit.compile', 'gcc.exec', 'llc.exec', 'llc-beta.exec', 'jit.exec') @@ -750,10 +750,10 @@ def append_to_sample_keys(tup): # for now, user time is the unqualified Time stat if config.test_time_stat == "real": time_stat = 'Real_' - append_to_sample_keys((True, 'compile', 'CC_' + time_stat + 'Time', + append_to_sample_keys((True, 'compile_time', 'CC_' + time_stat + 'Time', None, 'CC', float)) append_to_sample_keys((False, 'hash', 'CC_Hash', None, 'CC', str)) - append_to_sample_keys((True, 'exec', 'Exec_' + time_stat + 'Time', + append_to_sample_keys((True, 'execution_time', 'Exec_' + time_stat + 'Time', None, 'Exec', float)) else: test_namespace = 'nightlytest' diff --git a/lnt/tests/test_suite.py b/lnt/tests/test_suite.py index bb07c000..2c7083af 100644 --- a/lnt/tests/test_suite.py +++ b/lnt/tests/test_suite.py @@ -111,10 +111,7 @@ def _importProfile(objdump, name_filename): pf.upgrade() profilefile = pf.render() - return lnt.testing.TestSamples(name + '.profile', - [profilefile], - {}, - str) + return lnt.testing.MetricSamples(name + '.profile', [profilefile], conv_f=str) def _lit_json_to_template(json_reports, template_engine): @@ -723,8 +720,8 @@ def _get_cc_info(self, cmake_vars): def _parse_lit_output(self, path, data, cmake_vars, only_test=False): LIT_METRIC_TO_LNT = { - 'compile_time': 'compile', - 'exec_time': 'exec', + 'compile_time': 'compile_time', + 'exec_time': 'execution_time', 'score': 'score', 'hash': 'hash', 'link_time': 'compile', @@ -747,9 +744,7 @@ def _parse_lit_output(self, path, data, cmake_vars, only_test=False): 'link_mem_bytes': float } - # We don't use the test info, currently. - test_info = {} - test_samples = [] + tests = [] # FIXME: Populate with keys not to upload ignore = self.opts.exclude_stat_from_submission @@ -762,6 +757,7 @@ def _parse_lit_output(self, path, data, cmake_vars, only_test=False): for test_data in data['tests']: code = test_data['code'] raw_name = test_data['name'] + metric_samples = [] split_name = raw_name.split(' :: ', 1) if len(split_name) > 1: @@ -796,33 +792,27 @@ def _parse_lit_output(self, path, data, cmake_vars, only_test=False): if k not in LIT_METRIC_TO_LNT or \ LIT_METRIC_TO_LNT[k] in ignore: continue - server_name = name + '.' + LIT_METRIC_TO_LNT[k] if k == 'link_time' or k == 'link_mem_bytes': # Move link time into a second benchmark's # compile-time. + # TODO: This is more difficult with the new API server_name = name + '-link.' + LIT_METRIC_TO_LNT[k] - test_samples.append( - lnt.testing.TestSamples(server_name, - [v], - test_info, - LIT_METRIC_CONV_FN[k])) + sample = lnt.testing.MetricSamples(LIT_METRIC_TO_LNT[k], [v], conv_f=LIT_METRIC_CONV_FN[k]) + metric_samples.append(sample) if code == 'NOEXE': - test_samples.append( - lnt.testing.TestSamples(name + '.compile.status', - [lnt.testing.FAIL], - test_info)) + metric_samples.append(lnt.testing.MetricSamples('compile.status', [lnt.testing.FAIL])) no_errors = False elif not is_pass: lnt_code = self._get_lnt_code(test_data['code']) - test_samples.append( - lnt.testing.TestSamples(name + '.exec.status', - [lnt_code], test_info)) + metric_samples.append(lnt.testing.MetricSamples('exec.status', [lnt_code])) no_errors = False + tests.append(lnt.testing.Test(name, metric_samples)) + # Now import the profiles in parallel. if profiles_to_import: logger.info('Importing %d profiles with %d threads...' % @@ -833,9 +823,7 @@ def _parse_lit_output(self, path, data, cmake_vars, only_test=False): func = partial(_importProfile, cmake_vars["CMAKE_OBJDUMP"]) waiter = pool.map_async(func, profiles_to_import) samples = waiter.get(TIMEOUT) - test_samples.extend([sample - for sample in samples - if sample is not None]) + metric_samples.extend([sample for sample in samples if sample is not None]) except multiprocessing.TimeoutError: logger.warning('Profiles had not completed importing after ' + '%s seconds.' % TIMEOUT) @@ -846,22 +834,17 @@ def _parse_lit_output(self, path, data, cmake_vars, only_test=False): raise RuntimeError("Result %s did not exist!" % self.opts.single_result) - # FIXME: Add more machine info! - run_info = { - 'tag': 'nts', - 'no_errors': no_errors, - } + run_info = {'no_errors': no_errors} run_info.update(self._get_cc_info(cmake_vars)) run_info['run_order'] = run_info['inferred_run_order'] if self.opts.run_order: run_info['run_order'] = self.opts.run_order + run_info['llvm_project_revision'] = run_info['run_order'] - machine_info = { - } - - machine = lnt.testing.Machine(self.opts.label, machine_info) + # FIXME: Add more machine info! + machine = lnt.testing.Machine(self.opts.label, info={}) run = lnt.testing.Run(self.start_time, timestamp(), info=run_info) - report = lnt.testing.Report(machine, run, test_samples) + report = lnt.testing.Report(machine, run, tests) return report def _unix_quote_args(self, s): diff --git a/tests/runtest/exclude_stat.py b/tests/runtest/exclude_stat.py index 9467466e..785dabc2 100644 --- a/tests/runtest/exclude_stat.py +++ b/tests/runtest/exclude_stat.py @@ -10,5 +10,6 @@ # RUN: filecheck --check-prefix CHECK-STDOUT < %t.log %s # RUN: filecheck --check-prefix CHECK-REPORT < %t.SANDBOX/build/report.json %s # CHECK-STDOUT: Import succeeded. -# CHECK-REPORT: "Name": "nts.{{[^.]+}}.exec" -# CHECK-REPORT-NOT: "Name": "nts.{{[^.]+}}.compile" +# CHECK-REPORT: "execution_time": +# CHECK-REPORT-NOT: "compile_time": +# CHECK-REPORT-NOT: "compile": diff --git a/tests/runtest/test_suite-fail-compile.shtest b/tests/runtest/test_suite-fail-compile.shtest index 1998e8f2..6502a95b 100644 --- a/tests/runtest/test_suite-fail-compile.shtest +++ b/tests/runtest/test_suite-fail-compile.shtest @@ -12,4 +12,5 @@ # RUN: filecheck --check-prefix CHECK-RESULTS-FAIL-COMPILE < %t.SANDBOX/build/report.json %s # CHECK-RESULTS-FAIL-COMPILE: "no_errors": "False" # CHECK-RESULTS-FAIL-COMPILE: "run_order": "123" -# CHECK-RESULTS-FAIL-COMPILE: "Name": "nts.bar.compile.status" +# CHECK-RESULTS-FAIL-COMPILE: "compile.status": 1.0 +# CHECK-RESULTS-FAIL-COMPILE: "name": "nts.bar" diff --git a/tests/runtest/test_suite-fail-exec.shtest b/tests/runtest/test_suite-fail-exec.shtest index 912161fe..66534d90 100644 --- a/tests/runtest/test_suite-fail-exec.shtest +++ b/tests/runtest/test_suite-fail-exec.shtest @@ -12,4 +12,5 @@ # RUN: filecheck --check-prefix CHECK-RESULTS-FAIL-EXEC < %t.SANDBOX/build/report.json %s # CHECK-RESULTS-FAIL-EXEC: "no_errors": "False" # CHECK-RESULTS-FAIL-EXEC: "run_order": "123" -# CHECK-RESULTS-FAIL-EXEC: "Name": "nts.baz.exec.status" +# CHECK-RESULTS-FAIL-EXEC: "exec.status": 1.0 +# CHECK-RESULTS-FAIL-EXEC: "name": "nts.baz" diff --git a/tests/runtest/test_suite-metrics.shtest b/tests/runtest/test_suite-metrics.shtest index aee72268..f0d67c1c 100644 --- a/tests/runtest/test_suite-metrics.shtest +++ b/tests/runtest/test_suite-metrics.shtest @@ -13,8 +13,9 @@ # RUN: > %t.log 2> %t.err # RUN: filecheck --check-prefix CHECK-METRICS < %t.SANDBOX/build/report.json %s # RUN: filecheck --check-prefix CHECK-METRICS2 < %t.SANDBOX/build/report.json %s -# CHECK-METRICS-DAG: foo.exec -# CHECK-METRICS-DAG: foo.compile -# CHECK-METRICS-DAG: foo.score -# CHECK-METRICS-DAG: foo.hash -# CHECK-METRICS2-NOT: foo.unknown +# CHECK-METRICS-DAG: "name": "nts.foo" +# CHECK-METRICS-DAG: "execution_time": +# CHECK-METRICS-DAG: "compile_time": +# CHECK-METRICS-DAG: "score": +# CHECK-METRICS-DAG: "hash": +# CHECK-METRICS2-NOT: "unknown": diff --git a/tests/runtest/test_suite.shtest b/tests/runtest/test_suite.shtest index 5226b87d..0119a1aa 100644 --- a/tests/runtest/test_suite.shtest +++ b/tests/runtest/test_suite.shtest @@ -19,10 +19,11 @@ # CHECK-REPORT: "no_errors": "True", # CHECK-REPORT: "run_order": "154331" -# CHECK-REPORT: "Name": "nts.foo.compile" -# CHECK-REPORT: "Name": "nts.foo.exec" -# CHECK-REPORT: "Name": "nts.foo.hash" -# CHECK-REPORT: "Name": "nts.foo.score" +# CHECK-REPORT: "compile_time": 1.3 +# CHECK-REPORT: "execution_time": 1.4 +# CHECK-REPORT: "hash": "xyz" +# CHECK-REPORT: "name": "nts.foo" +# CHECK-REPORT: "score": 1.5 # # CHECK-BASIC: Inferred C++ compiler under test # CHECK-BASIC: Configuring diff --git a/tests/testing/TestingTest.py b/tests/testing/TestingTest.py index 72811a57..2b4bdd79 100644 --- a/tests/testing/TestingTest.py +++ b/tests/testing/TestingTest.py @@ -2,7 +2,6 @@ import unittest import logging -import copy import re import sys from datetime import datetime @@ -12,657 +11,320 @@ class MetricSamplesTest(unittest.TestCase): - def setUp(self): - self.samples_list_float = MetricSamples('execution_time', [21.4, 3.2]) - def test_constructor(self): # Check implicit float conversion from string. - samples_list_float_str = MetricSamples('execution_time', - ['6.7', '30.4']) - self.assertEqual(samples_list_float_str.metric, 'execution_time') - self.assertListEqual(samples_list_float_str.data, [6.7, 30.4]) - self.assertEqual(samples_list_float_str.report_version, 2) + samples = MetricSamples('execution_time', ['6.7', '30.4']) + self.assertEqual(samples.metric, 'execution_time') + self.assertListEqual(samples.data, [6.7, 30.4]) # Check explicit float conversion. - float_samples_list_float_str = MetricSamples('execution_time', - ['4.7', '32.4'], float) - self.assertEqual(float_samples_list_float_str.metric, 'execution_time') - self.assertListEqual(float_samples_list_float_str.data, [4.7, 32.4]) - self.assertEqual(float_samples_list_float_str.report_version, 2) + samples = MetricSamples('execution_time', ['4.7', '32.4'], float) + self.assertEqual(samples.metric, 'execution_time') + self.assertListEqual(samples.data, [4.7, 32.4]) # Check nop implicit float conversion from float. - self.assertEqual(self.samples_list_float.metric, 'execution_time') - self.assertListEqual(self.samples_list_float.data, [21.4, 3.2]) - self.assertEqual(self.samples_list_float.report_version, 2) + samples = MetricSamples('execution_time', [21.4, 3.2]) + self.assertEqual(samples.metric, 'execution_time') + self.assertListEqual(samples.data, [21.4, 3.2]) # Check implicit float conversion from integer. - samples_list_int = MetricSamples('execution_time', [6, 11]) - self.assertEqual(samples_list_int.metric, 'execution_time') - self.assertIsInstance(samples_list_int.data[0], float) - self.assertIsInstance(samples_list_int.data[1], float) - self.assertListEqual(samples_list_int.data, [6.0, 11.0]) - self.assertEqual(samples_list_int.report_version, 2) + samples = MetricSamples('execution_time', [6, 11]) + self.assertEqual(samples.metric, 'execution_time') + self.assertIsInstance(samples.data[0], float) + self.assertIsInstance(samples.data[1], float) + self.assertListEqual(samples.data, [6.0, 11.0]) # Check non-float conversion from float. - int_samples_list_float = MetricSamples('execution_time', [21.4, 3.2], - int) - self.assertEqual(int_samples_list_float.metric, 'execution_time') - self.assertIsInstance(int_samples_list_float.data[0], int) - self.assertIsInstance(int_samples_list_float.data[1], int) - self.assertListEqual(int_samples_list_float.data, [21, 3]) - self.assertEqual(int_samples_list_float.report_version, 2) + samples = MetricSamples('execution_time', [21.4, 3.2], int) + self.assertEqual(samples.metric, 'execution_time') + self.assertIsInstance(samples.data[0], int) + self.assertIsInstance(samples.data[1], int) + self.assertListEqual(samples.data, [21, 3]) # Check non-float conversion from same input type. - int_samples_list_int = MetricSamples('execution_time', [6, 11], int) - self.assertEqual(int_samples_list_int.metric, 'execution_time') - self.assertIsInstance(int_samples_list_int.data[0], int) - self.assertIsInstance(int_samples_list_int.data[1], int) - self.assertListEqual(int_samples_list_int.data, [6, 11]) - self.assertEqual(int_samples_list_int.report_version, 2) + samples = MetricSamples('execution_time', [6, 11], int) + self.assertEqual(samples.metric, 'execution_time') + self.assertIsInstance(samples.data[0], int) + self.assertIsInstance(samples.data[1], int) + self.assertListEqual(samples.data, [6, 11]) # Check explicit version. - samples_list_float_version = MetricSamples('execution_time', - [22.4, 5.2], - report_version=2) - self.assertEqual(samples_list_float_version.metric, 'execution_time') - self.assertListEqual(samples_list_float_version.data, [22.4, 5.2]) - self.assertEqual(samples_list_float_version.report_version, 2) - - # Check call to check(). - self.assertRaises(AssertionError, MetricSamples, 'execution_time', - [22.4, 5.2], report_version=1) - - def test_check(self): - # Check valid instance. - self.samples_list_float.report_version = 2 - self.samples_list_float.check() - - # Check too small version. - self.samples_list_float.report_version = 1 - self.assertRaises(AssertionError, self.samples_list_float.check) - - # Check too big version. - self.samples_list_float.report_version = 3 - self.assertRaises(AssertionError, self.samples_list_float.check) + samples = MetricSamples('execution_time', [22.4, 5.2]) + self.assertEqual(samples.metric, 'execution_time') + self.assertListEqual(samples.data, [22.4, 5.2]) def test_add_samples(self): + samples = MetricSamples('execution_time', [21.4, 3.2]) + # Check nop implicit float conversion from float. - self.samples_list_float.add_samples([9.9]) - self.assertListEqual(self.samples_list_float.data, [21.4, 3.2, 9.9]) + samples.add_samples([9.9]) + self.assertListEqual(samples.data, [21.4, 3.2, 9.9]) # Check implicit float conversion from string. - self.samples_list_float.add_samples(['4.4']) - self.assertListEqual(self.samples_list_float.data, - [21.4, 3.2, 9.9, 4.4]) + samples.add_samples(['4.4']) + self.assertListEqual(samples.data, [21.4, 3.2, 9.9, 4.4]) # Check explicit float conversion from integer. - self.samples_list_float.add_samples([2]) - self.assertIsInstance(self.samples_list_float.data[-1], float) - self.assertListEqual(self.samples_list_float.data, - [21.4, 3.2, 9.9, 4.4, 2.0]) + samples.add_samples([2]) + self.assertIsInstance(samples.data[-1], float) + self.assertListEqual(samples.data, [21.4, 3.2, 9.9, 4.4, 2.0]) # Check int conversion from float. - self.samples_list_float.add_samples([11.6], int) - self.assertListEqual(self.samples_list_float.data, - [21.4, 3.2, 9.9, 4.4, 2.0, 11]) + samples.add_samples([11.6], int) + self.assertListEqual(samples.data, [21.4, 3.2, 9.9, 4.4, 2.0, 11]) def test_render(self): # Check rendering with several samples. - self.assertDictEqual(self.samples_list_float.render(), - dict(execution_time=[21.4, 3.2])) + samples = MetricSamples('execution_time', [21.4, 3.2]) + self.assertDictEqual(samples.render(), dict(execution_time=[21.4, 3.2])) # Check rendering with a single sample. - samples_list_one_float = MetricSamples('execution_time', [7.3]) - self.assertDictEqual(samples_list_one_float.render(), - dict(execution_time=7.3)) + samples = MetricSamples('execution_time', [7.3]) + self.assertDictEqual(samples.render(), dict(execution_time=7.3)) class TestTest(unittest.TestCase): def setUp(self): self.samples = [MetricSamples('execution_time', [21.4, 3.2])] - self.test_noinfo = Test('Test1', self.samples) - self.test_info = Test('Test2', self.samples, {'nb_files': 2}) def test_constructor(self): - # Check default version, no extra info. - self.assertEqual(self.test_noinfo.name, 'Test1') - self.assertListEqual(self.test_noinfo.samples, self.samples) - self.assertDictEqual(self.test_noinfo.info, dict()) - self.assertEqual(self.test_noinfo.report_version, 2) - - # Check default version, extra info. - self.assertEqual(self.test_info.name, 'Test2') - self.assertListEqual(self.test_info.samples, self.samples) - self.assertDictEqual(self.test_info.info, dict(nb_files='2')) - self.assertEqual(self.test_info.report_version, 2) - - # Check explicit version, no extra info. - test_noinfo_version = Test('Test3', self.samples, report_version=2) - self.assertListEqual(test_noinfo_version.samples, self.samples) - self.assertDictEqual(test_noinfo_version.info, dict()) - self.assertEqual(test_noinfo_version.report_version, 2) - - # Check call to check(). - self.assertRaises(AssertionError, Test, 'Test4', self.samples, - report_version=1) - - def test_check(self): - # Check too small version. - self.test_noinfo.report_version = 1 - self.assertRaises(AssertionError, self.test_noinfo.check) - - # Check too big version. - self.test_noinfo.report_version = 3 - self.assertRaises(AssertionError, self.test_noinfo.check) - - # Check valid instance. - self.test_noinfo.report_version = 2 - self.test_noinfo.check() - - # Check wrong instance for tests. - self.test_noinfo.samples = [self.samples[0], 2] - self.assertRaises(AssertionError, self.test_noinfo.check) + # Check without extra info. + test = Test('Test1', self.samples) + self.assertEqual(test.name, 'Test1') + self.assertListEqual(test.samples, self.samples) + self.assertDictEqual(test.info, dict()) + + # Check with extra info. + test = Test('Test2', self.samples, {'nb_files': '2'}) + self.assertEqual(test.name, 'Test2') + self.assertListEqual(test.samples, self.samples) + self.assertDictEqual(test.info, {'nb_files': '2'}) def test_render(self): # Check rendering with no info. - d1 = {'Name': 'Test1', - 'execution_time': [21.4, 3.2]} - self.assertDictEqual(self.test_noinfo.render(), d1) + test = Test('Test1', self.samples) + d = {'name': 'Test1', 'execution_time': [21.4, 3.2]} + self.assertDictEqual(test.render(), d) # Check rendering with info. - d2 = {'Name': 'Test2', - 'execution_time': [21.4, 3.2], - 'nb_files': '2'} - self.assertDictEqual(self.test_info.render(), d2) + test = Test('Test2', self.samples, {'nb_files': 2}) + d = {'name': 'Test2', 'execution_time': [21.4, 3.2], 'nb_files': '2'} + self.assertDictEqual(test.render(), d) class TestTestSamples(unittest.TestCase): - def setUp(self): - self.test_samples_int_list_noinfo = TestSamples('Test1', [1, 2]) - self.test_samples_str_float_list_noinfo = TestSamples('Test3', - ['2.3', '5.8']) - self.test_samples_int_list_info = TestSamples('Test7', [1.7, 2.8], - {'nb_files': 2}) - def test_constructor(self): # Check implicit float conversion from integer. - self.assertEqual(self.test_samples_int_list_noinfo.name, 'Test1') - self.assertDictEqual(self.test_samples_int_list_noinfo.info, {}) - self.assertIsInstance(self.test_samples_int_list_noinfo.data[0], float) - self.assertIsInstance(self.test_samples_int_list_noinfo.data[1], float) - self.assertListEqual(self.test_samples_int_list_noinfo.data, - [1.0, 2.0]) + samples = TestSamples('Test1', [1, 2]) + self.assertEqual(samples.name, 'Test1') + self.assertDictEqual(samples.info, {}) + self.assertIsInstance(samples.data[0], float) + self.assertIsInstance(samples.data[1], float) + self.assertListEqual(samples.data, [1.0, 2.0]) # Check explicit float conversion from integer. - float_test_samples_int_list_noinfo = TestSamples('Test2', [8, 9], - conv_f=float) - self.assertEqual(float_test_samples_int_list_noinfo.name, 'Test2') - self.assertDictEqual(float_test_samples_int_list_noinfo.info, {}) - self.assertIsInstance(float_test_samples_int_list_noinfo.data[0], - float) - self.assertIsInstance(float_test_samples_int_list_noinfo.data[1], - float) - self.assertListEqual(float_test_samples_int_list_noinfo.data, - [8.0, 9.0]) + samples = TestSamples('Test2', [8, 9], conv_f=float) + self.assertEqual(samples.name, 'Test2') + self.assertDictEqual(samples.info, {}) + self.assertIsInstance(samples.data[0], float) + self.assertIsInstance(samples.data[1], float) + self.assertListEqual(samples.data, [8.0, 9.0]) # Check implicit float conversion from string. - self.assertEqual(self.test_samples_str_float_list_noinfo.name, 'Test3') - self.assertDictEqual(self.test_samples_str_float_list_noinfo.info, {}) - self.assertListEqual(self.test_samples_str_float_list_noinfo.data, - [2.3, 5.8]) + samples = TestSamples('Test3', ['2.3', '5.8']) + self.assertEqual(samples.name, 'Test3') + self.assertDictEqual(samples.info, {}) + self.assertListEqual(samples.data, [2.3, 5.8]) # Check implicit nop float conversion from float. - test_samples_float_list_noinfo = TestSamples('Test4', [6.4, 5.5]) - self.assertEqual(test_samples_float_list_noinfo.name, 'Test4') - self.assertDictEqual(test_samples_float_list_noinfo.info, {}) - self.assertListEqual(test_samples_float_list_noinfo.data, [6.4, 5.5]) + samples = TestSamples('Test4', [6.4, 5.5]) + self.assertEqual(samples.name, 'Test4') + self.assertDictEqual(samples.info, {}) + self.assertListEqual(samples.data, [6.4, 5.5]) # Check nop non-float conversion from input of same type. - int_test_samples_int_list_noinfo = TestSamples('Test5', [1, 2], - conv_f=int) - self.assertEqual(int_test_samples_int_list_noinfo.name, 'Test5') - self.assertDictEqual(int_test_samples_int_list_noinfo.info, {}) - self.assertListEqual(int_test_samples_int_list_noinfo.data, [1, 2]) + samples = TestSamples('Test5', [1, 2], conv_f=int) + self.assertEqual(samples.name, 'Test5') + self.assertDictEqual(samples.info, {}) + self.assertListEqual(samples.data, [1, 2]) # Check non-float conversion from string. - int_test_samples_float_list_noinfo = TestSamples('Test6', [6.4, 5.5], - conv_f=int) - self.assertEqual(int_test_samples_float_list_noinfo.name, 'Test6') - self.assertDictEqual(int_test_samples_float_list_noinfo.info, {}) - self.assertListEqual(int_test_samples_float_list_noinfo.data, [6, 5]) + samples = TestSamples('Test6', [6.4, 5.5], conv_f=int) + self.assertEqual(samples.name, 'Test6') + self.assertDictEqual(samples.info, {}) + self.assertListEqual(samples.data, [6, 5]) # Check non-float conversion from float. - self.assertEqual(self.test_samples_int_list_info.name, 'Test7') - self.assertDictEqual(self.test_samples_int_list_info.info, - {'nb_files': '2'}) - self.assertListEqual(self.test_samples_int_list_info.data, [1.7, 2.8]) + samples = TestSamples('Test7', [1.7, 2.8], {'nb_files': 2}) + self.assertEqual(samples.name, 'Test7') + self.assertDictEqual(samples.info, {'nb_files': '2'}) + self.assertListEqual(samples.data, [1.7, 2.8]) def test_render(self): # Check rendering with no info. - d1 = {'Name': 'Test3', - 'Info': {}, - 'Data': [2.3, 5.8]} - self.assertDictEqual(self.test_samples_str_float_list_noinfo.render(), - d1) + samples = TestSamples('Test1', ['2.3', '5.8']) + d = {'Name': 'Test1', 'Info': {}, 'Data': [2.3, 5.8]} + self.assertDictEqual(samples.render(), d) # Check rendering with info. - d2 = {'Name': 'Test7', - 'Info': {'nb_files': '2'}, - 'Data': [1.7, 2.8]} - self.assertDictEqual(self.test_samples_int_list_info.render(), d2) + samples = TestSamples('Test2', [1.7, 2.8], {'nb_files': 2}) + d = {'Name': 'Test2', 'Info': {'nb_files': '2'}, 'Data': [1.7, 2.8]} + self.assertDictEqual(samples.render(), d) class TestRun(unittest.TestCase): - def setUp(self): - self.info_v1 = {'tag': 'nts', 'run_order': 18246} - self.run_float_start_v1 = Run(0.0, None, self.info_v1) - self.run_float_end_v1 = Run(None, 0.0, self.info_v1) - - self.info_v2 = {'llvm_project_revision': 18246} - self.run_float_start_v2 = Run(0.0, info=self.info_v2, report_version=2) - self.run_float_end_v2 = Run(end_time=0.0, info=self.info_v2, - report_version=2) - def test_constructor(self): - info = {'__report_version__': '1', - 'tag': 'nts', - 'run_order': '18246'} - - # Check time normalization of end time from float. - self.assertEqual(self.run_float_start_v1.start_time, - '1970-01-01 00:00:00') - self.assertTrue(self.run_float_start_v1.end_time) - self.assertNotEqual(self.run_float_start_v1.end_time, - self.run_float_start_v1.start_time) - self.assertTrue(datetime.strptime(self.run_float_start_v1.end_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(self.run_float_start_v1.info, info) - self.assertEqual(self.run_float_start_v1.report_version, 1) - - # Check time normalization of end time from datetime. - run_str_start_v1 = Run('2019-07-01 01:02:03', None, info=self.info_v1) - self.assertEqual(run_str_start_v1.start_time, '2019-07-01 01:02:03') - self.assertTrue(run_str_start_v1.end_time) - self.assertNotEqual(run_str_start_v1.end_time, - run_str_start_v1.start_time) - self.assertTrue(datetime.strptime(run_str_start_v1.end_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(run_str_start_v1.info, info) - self.assertEqual(run_str_start_v1.report_version, 1) - - # Check time normalization of end time from string. - run_datetime_start_v1 = Run(datetime(2019, 7, 2), None, - info=self.info_v1) - self.assertEqual(run_datetime_start_v1.start_time, - '2019-07-02 00:00:00') - self.assertTrue(run_datetime_start_v1.end_time) - self.assertNotEqual(run_datetime_start_v1.end_time, - run_datetime_start_v1.start_time) - self.assertTrue(datetime.strptime(run_datetime_start_v1.end_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(run_datetime_start_v1.info, info) - self.assertEqual(run_datetime_start_v1.report_version, 1) + info = {'llvm_project_revision': '18246'} # Check time normalization of start time from float. - run_float_end_v1 = Run(None, 0.0, self.info_v1) - self.assertEqual(run_float_end_v1.end_time, '1970-01-01 00:00:00') - self.assertTrue(run_float_end_v1.start_time) - self.assertNotEqual(run_float_end_v1.start_time, - run_float_end_v1.end_time) - self.assertTrue(datetime.strptime(run_float_end_v1.start_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(run_float_end_v1.info, info) - self.assertEqual(run_float_end_v1.report_version, 1) - - # Check time normalization of start time from datetime. - run_str_end_v1 = Run(None, '2019-07-01 01:02:03', self.info_v1) - self.assertEqual(run_str_end_v1.end_time, '2019-07-01 01:02:03') - self.assertTrue(run_str_end_v1.start_time) - self.assertNotEqual(run_str_end_v1.start_time, run_str_end_v1.end_time) - self.assertTrue(datetime.strptime(run_str_end_v1.start_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(run_str_end_v1.info, info) - self.assertEqual(run_str_end_v1.report_version, 1) + run = Run(0.0, info=info) + self.assertEqual(run.start_time, '1970-01-01 00:00:00') + self.assertIsNone(run.end_time) + self.assertDictEqual(run.info, info) # Check time normalization of start time from string. - run_datetime_end_v1 = Run(None, datetime(2019, 7, 2), self.info_v1) - self.assertEqual(run_datetime_end_v1.end_time, '2019-07-02 00:00:00') - self.assertTrue(run_datetime_end_v1.start_time) - self.assertNotEqual(run_datetime_end_v1.start_time, - run_datetime_end_v1.end_time) - self.assertTrue(datetime.strptime(run_datetime_end_v1.start_time, - '%Y-%m-%d %H:%M:%S')) - self.assertDictEqual(run_datetime_end_v1.info, info) - self.assertEqual(run_datetime_end_v1.report_version, 1) - - # Check failure when info contains __report_version__ key. - self.assertRaisesRegex(ValueError, '__report_version__.*reserved', - Run, None, None, info) - - # Check missing tag entry in info for format version 1. - self.assertRaisesRegex(ValueError, - "Missing 'tag' entry in 'info' dictionary", - Run, info={'run_order': 40385}) - - # Check missing run_order entry in info for format version 1. - self.assertRaisesRegex(ValueError, - "Missing 'run_order' entry in 'info'" - " dictionary", Run, info={'tag': 'nts'}) - - # Test empty start and end time in format version 2 - self.assertEqual(self.run_float_start_v2.start_time, - '1970-01-01 00:00:00') - self.assertIsNone(self.run_float_start_v2.end_time) - self.assertDictEqual(self.run_float_start_v2.info, - {'llvm_project_revision': '18246'}) - self.assertEqual(self.run_float_start_v2.report_version, 2) - - # Check missing llvm_project_revision entry in info for format - # version 2. - self.assertRaisesRegex(ValueError, - "Missing 'llvm_project_revision' entry in" - " 'info' dictionary", Run, 0.0, info={}, - report_version=2) + run = Run('2019-07-01 01:02:03', None, info=info) + self.assertEqual(run.start_time, '2019-07-01 01:02:03') + self.assertIsNone(run.end_time) + self.assertDictEqual(run.info, info) - # Check call to check() - self.assertRaises(AssertionError, Run, info=self.info_v2, - report_version=3) - - def test_check(self): - # Check valid v1 instance. - self.run_float_start_v1.check() + # Check time normalization of start time from datetime. + run = Run(datetime(2019, 7, 2), None, info=info) + self.assertEqual(run.start_time, '2019-07-02 00:00:00') + self.assertIsNone(run.end_time) + self.assertDictEqual(run.info, info) - # Check too big version. - self.run_float_start_v2.report_version = 3 - self.assertRaises(AssertionError, self.run_float_start_v2.check) + # Check time normalization of end time from float. + run = Run(None, 0.0, info) + self.assertIsNone(run.start_time) + self.assertEqual(run.end_time, '1970-01-01 00:00:00') + self.assertDictEqual(run.info, info) - # Check valid v2 instance. - self.run_float_start_v2.report_version = 2 - self.run_float_start_v2.start_time = None - self.run_float_start_v2.check() + # Check time normalization of end time from string. + run = Run(None, '2019-07-01 01:02:03', info) + self.assertIsNone(run.start_time) + self.assertEqual(run.end_time, '2019-07-01 01:02:03') + self.assertDictEqual(run.info, info) - # Check no time or info. - self.run_float_start_v2.info = {} - self.assertRaisesRegex(ValueError, 'No data defined in this Run', - self.run_float_start_v2.check) + # Check time normalization of end time from datetime. + run = Run(None, datetime(2019, 7, 2), info) + self.assertIsNone(run.start_time) + self.assertEqual(run.end_time, '2019-07-02 00:00:00') + self.assertDictEqual(run.info, info) + + # Test empty start and end time. + run = Run(info=info) + self.assertIsNone(run.start_time) + self.assertIsNone(run.end_time) + self.assertDictEqual(run.info, info) + + # Check missing llvm_project_revision entry. + self.assertRaisesRegex(ValueError, + "Missing 'llvm_project_revision' entry in 'info' dictionary", Run, 0.0, info={}) def test_update(self): + info = {'llvm_project_revision': '18246'} + # Check update with a supplied end time. - end_time_updated_run_float_start_v1 = ( - copy.deepcopy(self.run_float_start_v1)) - end_time_updated_run_float_start_v1.update_endtime( - datetime(2019, 8, 2)) - self.assertEqual(end_time_updated_run_float_start_v1.end_time, - '2019-08-02 00:00:00') - - # Check update with default end time in format v1: end time = - # now. - updated_run_float_start_v1 = ( - copy.deepcopy(end_time_updated_run_float_start_v1)) - updated_run_float_start_v1.update_endtime() - self.assertTrue(updated_run_float_start_v1.end_time) - self.assertNotEqual(updated_run_float_start_v1.end_time, - updated_run_float_start_v1.start_time) - self.assertNotEqual(updated_run_float_start_v1.end_time, - end_time_updated_run_float_start_v1.end_time) - - # Check update with default end time in format v2: end time = - # None. - updated_run_float_end_v2 = copy.deepcopy(self.run_float_end_v2) - updated_run_float_end_v2.update_endtime() - self.assertEqual(updated_run_float_end_v2.start_time, - updated_run_float_end_v2.start_time) - self.assertIsNone(updated_run_float_end_v2.end_time) + run = Run(0.0, info=info) + run.update_endtime(datetime(2019, 8, 2)) + self.assertEqual(run.end_time, '2019-08-02 00:00:00') + + # Check update with default end time: end time = None. + run = Run(0.0, info=info) + self.assertIsNone(run.end_time) + run.update_endtime() + self.assertIsNone(run.end_time) def test_render(self): - # Check rendering of format v1. - d1 = {'Start Time': '1970-01-01 00:00:00', - 'End Time': self.run_float_start_v1.end_time, - 'Info': {'__report_version__': '1', - 'run_order': '18246', - 'tag': 'nts'}} - self.assertDictEqual(self.run_float_start_v1.render(), d1) + info = {'llvm_project_revision': '18246'} - # Check rendering of format v2 with no end time. - d2 = {'start_time': '1970-01-01 00:00:00', - 'llvm_project_revision': '18246'} - self.assertDictEqual(self.run_float_start_v2.render(), d2) + # Check rendering with start and end time. + run = Run('2019-07-01 01:02:03', '2019-07-01 04:00:00', info=info) + d = {'start_time': '2019-07-01 01:02:03', + 'end_time': '2019-07-01 04:00:00', + 'llvm_project_revision': '18246'} + self.assertDictEqual(run.render(), d) - # Check rendering of format v2 with no start time. - d3 = {'end_time': '1970-01-01 00:00:00', - 'llvm_project_revision': '18246'} - self.assertDictEqual(self.run_float_end_v2.render(), d3) + # Check rendering without start time. + run = Run(end_time='2019-07-01 04:00:00', info=info) + d = {'end_time': '2019-07-01 04:00:00', + 'llvm_project_revision': '18246'} + self.assertDictEqual(run.render(), d) + # Check rendering without end time. + run = Run('2019-07-01 01:02:03', info=info) + d = {'start_time': '2019-07-01 01:02:03', + 'llvm_project_revision': '18246'} + self.assertDictEqual(run.render(), d) -class TestMachine(unittest.TestCase): - def setUp(self): - self.machine_v1_noinfo = Machine('Machine1') - self.machine_v1 = Machine('Machine2', {'CPUs': 2}) - self.machine_v2 = Machine('Machine3', {'CPUs': 2}, 2) +class TestMachine(unittest.TestCase): def test_constructor(self): - # Check constructor with no info and default version (v1). - self.assertEqual(self.machine_v1_noinfo.name, 'Machine1') - self.assertDictEqual(self.machine_v1_noinfo.info, {}) - self.assertEqual(self.machine_v1_noinfo.report_version, 1) + info = {'CPUs': '2'} - # Check constructor with info and default version (v1). - self.assertEqual(self.machine_v1.name, 'Machine2') - self.assertDictEqual(self.machine_v1.info, {'CPUs': '2'}) - self.assertEqual(self.machine_v1.report_version, 1) + # Check constructor with no info. + m = Machine('Machine1') + self.assertEqual(m.name, 'Machine1') + self.assertDictEqual(m.info, {}) - # Check v2 constructor with info. - self.assertEqual(self.machine_v2.name, 'Machine3') - self.assertDictEqual(self.machine_v2.info, {'CPUs': '2'}) - self.assertEqual(self.machine_v2.report_version, 2) - - def test_check(self): - # Check valid v1 instance. - self.machine_v1.check() - - # Check valid v2 instance. - self.machine_v2.check() - - # Check too big version. - self.machine_v2.report_version = 3 - self.assertRaises(AssertionError, self.machine_v2.check) + # Check constructor with info. + m = Machine('Machine2', info=info) + self.assertEqual(m.name, 'Machine2') + self.assertDictEqual(m.info, info) def test_render(self): - # Check v1 rendering with no info. - d1 = {'Name': 'Machine1', - 'Info': {}} - self.assertDictEqual(self.machine_v1_noinfo.render(), d1) - - # Check v1 rendering with info. - d2 = {'Name': 'Machine2', - 'Info': {'CPUs': '2'}} - self.assertDictEqual(self.machine_v1.render(), d2) - - # Check v2 rendering with no info. - d3 = {'Name': 'Machine3', - 'CPUs': '2'} - self.assertDictEqual(self.machine_v2.render(), d3) + # Check rendering with no info. + m = Machine('Machine1') + d = {'name': 'Machine1'} + self.assertDictEqual(m.render(), d) - # Check v2 rendering with info. - self.machine_v2.info = {} - d4 = {'Name': 'Machine3'} - self.assertDictEqual(self.machine_v2.render(), d4) + # Check rendering with info. + m = Machine('Machine2', info={'CPUs': '2'}) + d = {'name': 'Machine2', 'CPUs': '2'} + self.assertDictEqual(m.render(), d) class TestReport(unittest.TestCase): maxDiff = None def setUp(self): - self.machine_v1 = Machine('Machine', info={'nb_cpus': 2}) - self.run_v1 = Run(0.0, '1982-01-01 00:00:00', {'tag': 'nts', - 'run_order': 18246}) - self.tests_samples = [TestSamples('Test.exec', [1.7, 2.8], - {'nb_files': 2})] - self.report_v1 = Report(self.machine_v1, self.run_v1, - self.tests_samples) - - self.machine_v2 = Machine('Machine', info={'nb_cpus': 2}, - report_version=2) - self.run_v2 = Run(0.0, info={'llvm_project_revision': 18246}, - report_version=2) + self.machine = Machine('Machine', info={'nb_cpus': 2}) + self.run = Run(0.0, info={'llvm_project_revision': '18246'}) samples = MetricSamples('execution_time', [21.4, 3.2]) self.tests = [Test('Test', [samples], {'nb_files': 2})] - self.report_v2 = Report(self.machine_v2, self.run_v2, self.tests, 2) def test_constructor(self): - # Check successful constructor call with default version. - self.assertEqual(self.report_v1.machine, self.machine_v1) - self.assertEqual(self.report_v1.run, self.run_v1) - self.assertListEqual(self.report_v1.tests, self.tests_samples) - self.assertEqual(self.report_v1.report_version, 1) - - # Check successful constructor call with explicit version. - self.assertEqual(self.report_v2.machine, self.machine_v2) - self.assertEqual(self.report_v2.run, self.run_v2) - self.assertListEqual(self.report_v2.tests, self.tests) - self.assertEqual(self.report_v2.report_version, 2) - - # Check call to check(). - self.assertRaises(AssertionError, Report, [], self.run_v1, - self.tests_samples) - - def test_check(self): - # Check wrong version. - self.report_v2.report_version = 3 - self.assertRaises(AssertionError, self.report_v2.check) - - # Check valid v2 report. - self.report_v2.report_version = 2 - self.report_v2.check() - - # Check type test for machine. - report_machine_list = copy.deepcopy(self.report_v1) - report_machine_list.machine = [] - self.assertRaises(AssertionError, report_machine_list.check) - - # Check version mismatch between machine and report. - self.report_v1.machine.report_version = 2 - self.assertRaises(AssertionError, self.report_v1.check) - - # Check valid v1 report. - self.report_v1.machine.report_version = 1 - self.report_v1.check() - - # Check type test for run. - report_run_list = copy.deepcopy(self.report_v1) - report_run_list.run = [] - self.assertRaises(AssertionError, report_run_list.check) - - # Check version mismatch between run and report. - self.report_v1.run.report_version = 2 - self.assertRaises(AssertionError, self.report_v1.check) - - self.report_v1.run.report_version = 1 - - # Check type test for all v1 tests. - report_v1_tests_int_list = copy.deepcopy(self.report_v1) - report_v1_tests_int_list.tests = [2] - self.assertRaises(AssertionError, report_v1_tests_int_list.check) - - # Check type test for all v2 tests. - report_v2_tests_int_list = copy.deepcopy(self.report_v2) - report_v2_tests_int_list.tests = [2] - self.assertRaises(AssertionError, report_v2_tests_int_list.check) - - # Check version mismatch between one of the tests and report. - self.report_v2.tests[0].report_version = 1 - self.assertRaises(AssertionError, self.report_v2.check) + # Check successful constructor call. + report = Report(self.machine, self.run, self.tests) + self.assertEqual(report.machine, self.machine) + self.assertEqual(report.run, self.run) + self.assertListEqual(report.tests, self.tests) def test_update_report(self): - # Check update with default (=now) end time. - orig_end_time = self.report_v1.run.end_time - new_tests_samples = [TestSamples('Test2.exec', [56.5])] - self.report_v1.update_report(new_tests_samples) - self.tests_samples.extend(new_tests_samples) - self.assertListEqual(self.report_v1.tests, self.tests_samples) - self.assertNotEqual(self.report_v1.run.end_time, orig_end_time) + # Check update with default (=None) end time. + report = Report(self.machine, self.run, self.tests) + new_samples = [Test('Test2.exec', [MetricSamples('execution_time', [56.5])])] + report.update_report(new_samples) + self.assertListEqual(report.tests, self.tests + new_samples) + self.assertIsNone(report.run.end_time) # Check update with supplied end time. - new_tests_samples = [TestSamples('Test3.exec', [18.3])] - self.report_v1.update_report(new_tests_samples, '1990-07-07 00:00:00') - self.tests_samples.extend(new_tests_samples) - self.assertListEqual(self.report_v1.tests, self.tests_samples) - self.assertEqual(self.report_v1.run.end_time, '1990-07-07 00:00:00') + report = Report(self.machine, self.run, self.tests) + new_samples = [Test('Test3.exec', [MetricSamples('execution_time', [18.3])])] + report.update_report(new_samples, '1990-07-07 00:00:00') + self.assertListEqual(report.tests, self.tests + new_samples) + self.assertEqual(report.run.end_time, '1990-07-07 00:00:00') def test_render(self): - # Check v1 format rendering with default indentation. - self.assertMultiLineEqual(re.sub(r' +\n', '\n', - self.report_v1.render()), """\ -{ - "Machine": { - "Info": { - "nb_cpus": "2" - }, - "Name": "Machine" - }, - "Run": { - "End Time": "1982-01-01 00:00:00", - "Info": { - "__report_version__": "1", - "run_order": "18246", - "tag": "nts" - }, - "Start Time": "1970-01-01 00:00:00" - }, - "Tests": [ - { - "Data": [ - 1.7, - 2.8 - ], - "Info": { - "nb_files": "2" - }, - "Name": "Test.exec" - } - ] -}""") - - # Check v1 format rendering with supplied indentation. - self.assertMultiLineEqual(re.sub(r' +\n', '\n', - self.report_v1.render(indent=2)), """\ -{ - "Machine": { - "Info": { - "nb_cpus": "2" - }, - "Name": "Machine" - }, - "Run": { - "End Time": "1982-01-01 00:00:00", - "Info": { - "__report_version__": "1", - "run_order": "18246", - "tag": "nts" - }, - "Start Time": "1970-01-01 00:00:00" - }, - "Tests": [ - { - "Data": [ - 1.7, - 2.8 - ], - "Info": { - "nb_files": "2" - }, - "Name": "Test.exec" - } - ] -}""") + report = Report(self.machine, self.run, self.tests) - # Check v2 format rendering with default indentation. - self.assertMultiLineEqual(re.sub(r' +\n', '\n', - self.report_v2.render()), """\ + # Check rendering with default indentation. + self.assertMultiLineEqual(re.sub(r' +\n', '\n', report.render()), """\ { "format_version": "2", "machine": { - "Name": "Machine", + "name": "Machine", "nb_cpus": "2" }, "run": { @@ -671,23 +333,22 @@ def test_render(self): }, "tests": [ { - "Name": "Test", "execution_time": [ 21.4, 3.2 ], + "name": "Test", "nb_files": "2" } ] }""") - # Check v2 format rendering with supplied indentation. - self.assertMultiLineEqual(re.sub(r' +\n', '\n', - self.report_v2.render(indent=2)), """\ + # Check rendering with supplied indentation. + self.assertMultiLineEqual(re.sub(r' +\n', '\n', report.render(indent=2)), """\ { "format_version": "2", "machine": { - "Name": "Machine", + "name": "Machine", "nb_cpus": "2" }, "run": { @@ -696,25 +357,24 @@ def test_render(self): }, "tests": [ { - "Name": "Test", "execution_time": [ 21.4, 3.2 ], + "name": "Test", "nb_files": "2" } ] }""") - # Check v2 format rendering with single sample for a metric and + # Check rendering with single sample for a metric and # default indentation. - self.report_v2.tests[0].samples[0].data.pop() - self.assertMultiLineEqual(re.sub(r' +\n', '\n', - self.report_v2.render()), """\ + report.tests[0].samples[0].data.pop() + self.assertMultiLineEqual(re.sub(r' +\n', '\n', report.render()), """\ { "format_version": "2", "machine": { - "Name": "Machine", + "name": "Machine", "nb_cpus": "2" }, "run": { @@ -723,8 +383,8 @@ def test_render(self): }, "tests": [ { - "Name": "Test", "execution_time": 21.4, + "name": "Test", "nb_files": "2" } ]