From 4f4d24c312065b65e7538224950ebc33e7926ec1 Mon Sep 17 00:00:00 2001 From: Edwin Takahashi Date: Thu, 16 Jul 2020 11:40:22 -0700 Subject: [PATCH 1/4] Changes: - implement the TestTask.overhead property that returns a float value representing non-group time. - add new query to obtain start/end times for the group and task. --- mozci/queries/test_task_overhead.query | 11 +++++++++++ mozci/task.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 mozci/queries/test_task_overhead.query diff --git a/mozci/queries/test_task_overhead.query b/mozci/queries/test_task_overhead.query new file mode 100644 index 00000000..c9512b0c --- /dev/null +++ b/mozci/queries/test_task_overhead.query @@ -0,0 +1,11 @@ +from: unittest +format: list +groupby: task.id +limit: 20000 +select: + - {value: action.start_time, name: task_min, aggregate: min} + - {value: action.end_time, name: task_max, aggregate: max} + - {value: result.start_time, name: group_min, aggregate: min} + - {value: result.end_time, name: group_max, aggregate: max} +where: + - in: {task.id: {$eval: task_id}} \ No newline at end of file diff --git a/mozci/task.py b/mozci/task.py index a91a4238..0abdb1d5 100644 --- a/mozci/task.py +++ b/mozci/task.py @@ -2,6 +2,7 @@ import json import os from abc import ABC, abstractmethod +from argparse import Namespace from dataclasses import dataclass, field from enum import Enum from inspect import signature @@ -9,6 +10,7 @@ from typing import Dict, List, Optional import requests +from adr.query import run_query from adr.util import memoized_property from loguru import logger from urllib3.response import HTTPResponse @@ -358,6 +360,30 @@ def configuration(self): parts = config.split("-") return "-".join(parts[:-1] if parts[-1].isdigit() else parts) + @property + def overhead(self): + """Calculate the overhead of a task. + + The methodology is simple: each task (action) has a start/end time. + Each group also has a start/end time. Take the earlist known group start + and latest known group end time, ensure the two falls somewhere in between + task start/end. + + Then calculate the overhead by taking the difference between the start + and end times. + + Returns: + float: difference between task start/end and group star/end times. + """ + data = run_query("test_task_overhead", Namespace(task_id=self.id))["data"].pop() + # Sanity check to ensure group start/end times are within task start/end. + assert data["task_min"] < data["group_min"] + assert data["task_max"] > data["group_max"] + + return (data["group_min"] - data["task_min"]) + ( + data["task_max"] - data["group_max"] + ) + # Don't perform type checking because of https://github.com/python/mypy/issues/5374. @dataclass # type: ignore From 15693e7bdfb5c39f69b38ee80f5c69326fee4279 Mon Sep 17 00:00:00 2001 From: Edwin Takahashi Date: Thu, 16 Jul 2020 17:27:58 -0700 Subject: [PATCH 2/4] Address code review commenst by @marco. - fix typos --- mozci/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mozci/task.py b/mozci/task.py index 0abdb1d5..b521fd1e 100644 --- a/mozci/task.py +++ b/mozci/task.py @@ -365,15 +365,15 @@ def overhead(self): """Calculate the overhead of a task. The methodology is simple: each task (action) has a start/end time. - Each group also has a start/end time. Take the earlist known group start + Each group also has a start/end time. Take the earliest known group start and latest known group end time, ensure the two falls somewhere in between task start/end. - Then calculate the overhead by taking the difference between the start - and end times. + This definition of overhead does not take into account inter-group overhead + eg. restarting browser, teardown, etc. Returns: - float: difference between task start/end and group star/end times. + float: difference between task start/end and group start/end times. """ data = run_query("test_task_overhead", Namespace(task_id=self.id))["data"].pop() # Sanity check to ensure group start/end times are within task start/end. From dd3edde4f743a8288542a8c03d3ca5b66c82e6b3 Mon Sep 17 00:00:00 2001 From: Edwin Takahashi Date: Fri, 17 Jul 2020 11:35:19 -0700 Subject: [PATCH 3/4] Changes: - implement overhead-related methods for LabelSummary --- mozci/task.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mozci/task.py b/mozci/task.py index b521fd1e..61a80a03 100644 --- a/mozci/task.py +++ b/mozci/task.py @@ -522,6 +522,18 @@ def total_duration(self): def median_duration(self): return median(self.durations) + @property + def overheads(self): + return [task.overhead for task in self.tasks] + + @property + def total_overheads(self): + return sum(self.overheads) + + @property + def median_overhead(self): + return median(self.overheads) + @memoized_property def status(self): overall_status = None From 64559bd720b9ef1638f6f95dd83879e4ef59efb6 Mon Sep 17 00:00:00 2001 From: Edwin Takahashi Date: Fri, 17 Jul 2020 11:43:06 -0700 Subject: [PATCH 4/4] Address code review comments from @klahnakoski - add overhead related abstract properties to GroupSummary, RunnableSummary - GroupSummary overhead will currently return 0 --- mozci/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mozci/task.py b/mozci/task.py index 61a80a03..1fa451d4 100644 --- a/mozci/task.py +++ b/mozci/task.py @@ -377,8 +377,8 @@ def overhead(self): """ data = run_query("test_task_overhead", Namespace(task_id=self.id))["data"].pop() # Sanity check to ensure group start/end times are within task start/end. - assert data["task_min"] < data["group_min"] - assert data["task_max"] > data["group_max"] + if data["task_min"] < data["group_min"] or data["task_max"] > data["group_max"]: + logger.warning(f"task f{self.id} has inconsistent group duration.") return (data["group_min"] - data["task_min"]) + ( data["task_max"] - data["group_max"]