Report timings for all tests

Instead of separating the benchmark tests from "normal" tests, store
timings for all successful tests and let the user decide which tests
they'd like to run via the -s option.

Author: carlosmn

clar.c

@@ -133,7 +133,7 @@ static struct {
 	int exit_on_error;
 	int report_suite_names;
 
-	int run_benchmarks;
+	int report_benchmarks;
 	double timing_start;
 	double timing_end;
 
@@ -157,7 +157,6 @@ static struct {
 struct clar_func {
 	const char *name;
 	void (*ptr)(void);
-	int is_bench;
 };
 
 struct clar_suite {
@@ -251,6 +250,8 @@ clar_run_test(
 	const struct clar_func *cleanup)
 {
 	_clar.test_status = CL_TEST_OK;
+	_clar.timing_start = 0.0;
+	_clar.timing_end = 0.0;
 	_clar.trampoline_enabled = 1;
 
 	CL_TRACE(CL_TRACE__TEST__BEGIN);
@@ -327,22 +328,19 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 		if (filter && strncmp(test[i].name, filter, matchlen))
 			continue;
 
-		if (test[i].is_bench != _clar.run_benchmarks)
-			continue;
-
 		_clar.active_test = test[i].name;
 		clar_run_test(&test[i], &suite->initialize, &suite->cleanup);
 
 		if (_clar.exit_on_error && _clar.total_errors)
 			return;
 
-		if (test[i].is_bench) {
-			clar_store_timing();
-		}
+		clar_store_timing();
 	}
 
-	puts("");
-	clar_report_timings();
+	if (_clar.report_benchmarks) {
+		puts("");
+		clar_report_timings();
+	}
 
 	_clar.active_test = NULL;
 	CL_TRACE(CL_TRACE__SUITE_END);
@@ -360,7 +358,7 @@ clar_usage(const char *arg)
 	printf("  -q    \tOnly report tests that had an error\n");
 	printf("  -Q    \tQuit as soon as a test fails\n");
 	printf("  -l    \tPrint suite names\n");
-	printf("  -b    \tRun benchmarks instead of tests\n");
+	printf("  -b    \tReport test benchmarks\n");
 	exit(-1);
 }
 
@@ -420,7 +418,7 @@ clar_parse_args(int argc, char **argv)
 		}
 
 		case 'b':
-			_clar.run_benchmarks = 1;
+			_clar.report_benchmarks = 1;
 			break;
 
 		case 'q':
@@ -509,7 +507,13 @@ clar_test(int argc, char **argv)
 
 static void clar_store_timing(void)
 {
-	struct clar_timing *timing = calloc(1, sizeof(struct clar_timing));
+	struct clar_timing *timing;
+
+	/* Failed tests jump over the timing code */
+	if (_clar.timing_end == 0)
+		return;
+
+	timing = calloc(1, sizeof(struct clar_timing));
 
 	if (_clar.timings == NULL)
 		_clar.timings = timing;

generate.py

@@ -18,7 +18,7 @@ def __init__(self, module):
         def _render_callback(self, cb):
             if not cb:
                 return '    { NULL, NULL }'
-            return '    { "%s", &%s, %d }' % (cb['short_name'], cb['symbol'], cb['bench'])
+            return '    { "%s", &%s }' % (cb['short_name'], cb['symbol'])
 
     class DeclarationTemplate(Template):
         def render(self):
@@ -79,13 +79,21 @@ def _replacer(match):
 
         return re.sub(SKIP_COMMENTS_REGEX, _replacer, text)
 
-    def _append_callbacks(self, callbacks, is_bench):
-        for (declaration, symbol, short_name) in callbacks:
+    def parse(self, contents):
+        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{"
+
+        contents = self._skip_comments(contents)
+        regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)
+
+        self.callbacks = []
+        self.initialize = None
+        self.cleanup = None
+
+        for (declaration, symbol, short_name) in regex.findall(contents):
             data = {
                 "short_name" : short_name,
                 "declaration" : declaration,
-                "symbol" : symbol,
-                "bench" : is_bench,
+                "symbol" : symbol
             }
 
             if short_name == 'initialize':
@@ -95,21 +103,6 @@ def _append_callbacks(self, callbacks, is_bench):
             else:
                 self.callbacks.append(data)
 
-    def parse(self, contents):
-        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{"
-        BENCH_FUNC_REGEX = r"^(void\s+(bench_%s__(\w+))\s*\(\s*void\s*\))\s*\{"
-
-        contents = self._skip_comments(contents)
-        test_regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)
-        bench_regex = re.compile(BENCH_FUNC_REGEX % self.name, re.MULTILINE)
-
-        self.callbacks = []
-        self.initialize = None
-        self.cleanup = None
-
-        self._append_callbacks(test_regex.findall(contents), False)
-        self._append_callbacks(bench_regex.findall(contents), True)
-
         return self.callbacks != []
 
     def refresh(self, path):

test/main.c

@@ -18,7 +18,6 @@
  */
 
 int global_test_counter = 0;
-int global_is_bench = 0;
 
 #ifdef _WIN32
 int __cdecl main(int argc, char *argv[])
@@ -35,10 +34,7 @@ int main(int argc, char *argv[])
 	ret = clar_test(argc, argv);
 
 	/* Your custom cleanup here */
-	if (global_is_bench)
-		cl_assert_equal_i(3, global_test_counter);
-	else
-		cl_assert_equal_i(8, global_test_counter);
+	cl_assert_equal_i(11, global_test_counter);
 
 	return ret;
 }

test/sample.c

@@ -83,34 +83,28 @@ void test_sample__ptr(void)
 	cl_assert_equal_p(&actual, actual);
 }
 
-void bench_sample__loop(void)
+void test_sample__bench_loop(void)
 {
 	int i;
 
-	global_is_bench = 1;
-
 	for (i = 0; i < 1000000; i++) {
 	}
 }
 
-void bench_sample__loop2(void)
+void test_sample__bench_loop2(void)
 {
 	int i;
 
-	global_is_bench = 1;
-
 	for (i = 0; i < 1000000; i++) {
 		int dummy = i*1000;
 		dummy = dummy;
 	}
 }
 
-void bench_sample__reset_timer(void)
+void test_sample__bench_reset_timer(void)
 {
 	int i;
 
-	global_is_bench = 1;
-
 	for (i = 0; i < 100000000; i++) {
 		int dummy = i*1000;
 		dummy = dummy;