diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 7ab4c73270d..7844b0b0112 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -39,6 +39,7 @@ on: - 'ci/scripts/util_*' - 'cpp/**' - 'compose.yaml' + - 'dev/archery/archery/**' - 'format/Flight.proto' - 'testing' tags: @@ -61,6 +62,7 @@ on: - 'ci/scripts/util_*' - 'cpp/**' - 'compose.yaml' + - 'dev/archery/archery/**' - 'format/Flight.proto' - 'testing' types: diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml index 3e4b7592153..1d2ae61f1eb 100644 --- a/.github/workflows/package_linux.yml +++ b/.github/workflows/package_linux.yml @@ -29,6 +29,7 @@ on: - '.github/workflows/report_ci.yml' - 'cpp/**' - 'c_glib/**' + - 'dev/archery/archery/**' - 'dev/release/binary-task.rb' - 'dev/release/verify-apt.sh' - 'dev/release/verify-yum.sh' @@ -43,6 +44,7 @@ on: - '.github/workflows/report_ci.yml' - 'cpp/**' - 'c_glib/**' + - 'dev/archery/archery/**' - 'dev/release/binary-task.rb' - 'dev/release/verify-apt.sh' - 'dev/release/verify-yum.sh' diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml index 687a4e0aa05..443d2354d7f 100644 --- a/.github/workflows/r_extra.yml +++ b/.github/workflows/r_extra.yml @@ -27,15 +27,16 @@ on: - '.github/workflows/check_labels.yml' - '.github/workflows/r_extra.yml' - '.github/workflows/report_ci.yml' - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "compose.yaml" - - "r/**" + - 'ci/docker/**' + - 'ci/etc/rprofile' + - 'ci/scripts/PKGBUILD' + - 'ci/scripts/cpp_*.sh' + - 'ci/scripts/install_minio.sh' + - 'ci/scripts/r_*.sh' + - 'cpp/**' + - 'compose.yaml' + - 'dev/archery/archery/**' + - 'r/**' tags: - '**' pull_request: @@ -44,15 +45,16 @@ on: - '.github/workflows/check_labels.yml' - '.github/workflows/r_extra.yml' - '.github/workflows/report_ci.yml' - - "ci/docker/**" - - "ci/etc/rprofile" - - "ci/scripts/PKGBUILD" - - "ci/scripts/cpp_*.sh" - - "ci/scripts/install_minio.sh" - - "ci/scripts/r_*.sh" - - "cpp/**" - - "compose.yaml" - - "r/**" + - 'ci/docker/**' + - 'ci/etc/rprofile' + - 'ci/scripts/PKGBUILD' + - 'ci/scripts/cpp_*.sh' + - 'ci/scripts/install_minio.sh' + - 'ci/scripts/r_*.sh' + - 'cpp/**' + - 'compose.yaml' + - 'dev/archery/archery/**' + - 'r/**' types: - labeled - opened diff --git a/dev/archery/archery/ci/cli.py b/dev/archery/archery/ci/cli.py index bf7b68d5327..5597dff733e 100644 --- a/dev/archery/archery/ci/cli.py +++ b/dev/archery/archery/ci/cli.py @@ -73,6 +73,22 @@ def report_chat(obj, workflow_id, send, repository, ignore, webhook, output.write(report_chat.render("workflow_report")) +class WorkflowEmailReport(EmailReport): + def __init__(self, **kwargs): + super().__init__('workflow_report', **kwargs) + + def date(self): + return self.report.datetime + + def subject(self): + workflow = self.report + date = self.date().strftime('%Y-%m-%d') + return ( + f'[{date}] Arrow Build Report for Job {workflow.name}: ' + f'{len(workflow.failed_jobs())} failed' + ) + + @ci.command() @click.argument('workflow_id', required=True) @click.option('--sender-name', '-n', @@ -105,9 +121,10 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, """ output = obj['output'] - email_report = EmailReport( - report=Workflow(workflow_id, repository, - ignore_job=ignore, gh_token=obj['github_token']), + workflow = Workflow(workflow_id, repository, + ignore_job=ignore, gh_token=obj['github_token']) + email_report = WorkflowEmailReport( + report=workflow, sender_name=sender_name, sender_email=sender_email, recipient_email=recipient_email @@ -119,8 +136,7 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email, smtp_password=smtp_password, smtp_server=smtp_server, smtp_port=smtp_port, - recipient_email=recipient_email, - message=email_report.render("workflow_report") + report=email_report ) else: - output.write(email_report.render("workflow_report")) + output.write(str(email_report.render())) diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py index c73c4d1ff7e..10aa3dedf44 100644 --- a/dev/archery/archery/crossbow/cli.py +++ b/dev/archery/archery/crossbow/cli.py @@ -343,6 +343,22 @@ def latest_prefix(obj, prefix, fetch): click.echo(latest.branch) +class NightlyEmailReport(EmailReport): + def __init__(self, **kwargs): + super().__init__('nightly_report', **kwargs) + + def subject(self): + report = self.report + n_errors = len(report.tasks_by_state['error']) + n_failures = len(report.tasks_by_state['failure']) + n_pendings = len(report.tasks_by_state['pending']) + return ( + f'[NIGHTLY] Arrow Build Report for Job {report.job.branch}: ' + f'{n_errors + n_failures} failed, ' + f'{n_pendings} pending' + ) + + @crossbow.command() @click.argument('job-name', required=True) @click.option('--sender-name', '-n', @@ -382,8 +398,9 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, queue.fetch() job = queue.get(job_name) - email_report = EmailReport( - report=Report(job), + report = Report(job) + email_report = NightlyEmailReport( + report=report, sender_name=sender_name, sender_email=sender_email, recipient_email=recipient_email @@ -401,11 +418,10 @@ def report(obj, job_name, sender_name, sender_email, recipient_email, smtp_password=smtp_password, smtp_server=smtp_server, smtp_port=smtp_port, - recipient_email=recipient_email, - message=email_report.render("nightly_report") + report=email_report ) else: - output.write(email_report.render("nightly_report")) + output.write(str(email_report.render())) @crossbow.command() @@ -601,6 +617,17 @@ def batch_gen(iterable, step): print(batch) +class TokenExpirationEmailReport(EmailReport): + def __init__(self, **kwargs): + super().__init__('token_expiration', **kwargs) + + def subject(self): + token_expiration_date = self.report.token_expiration_date + return ( + f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}' + ) + + @crossbow.command() @click.option('--days', default=30, help='Notification will be sent if expiration date is ' @@ -645,23 +672,18 @@ def __init__(self, token_expiration_date, days_left): self.token_expiration_date = token_expiration_date self.days_left = days_left - email_report = EmailReport( - report=TokenExpirationReport( - token_expiration_date or "ALREADY_EXPIRED", days_left), - sender_name=sender_name, - sender_email=sender_email, - recipient_email=recipient_email - ) + if not token_expiration_date: + token_expiration_date = 'ALREADY_EXPIRED' + report = TokenExpirationReport(token_expiration_date, days_left) + email_report = TokenExpirationEmailReport(report) - message = email_report.render("token_expiration").strip() if send: ReportUtils.send_email( smtp_user=smtp_user, smtp_password=smtp_password, smtp_server=smtp_server, smtp_port=smtp_port, - recipient_email=recipient_email, - message=message + report=email_report ) else: - output.write(message) + output.write(str(email_report.render())) diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py index 32962410d6e..a2c0487a2b1 100644 --- a/dev/archery/archery/crossbow/reports.py +++ b/dev/archery/archery/crossbow/reports.py @@ -17,6 +17,10 @@ import collections import csv +import datetime +import email.headerregistry +import email.message +import email.utils import operator import fnmatch import functools @@ -246,7 +250,7 @@ def send_message(cls, webhook, message): @classmethod def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port, - recipient_email, message): + report): from smtplib import SMTP, SMTP_SSL if smtp_port == 465: @@ -259,7 +263,8 @@ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port, else: smtp.starttls() smtp.login(smtp_user, smtp_password) - smtp.sendmail(smtp_user, recipient_email, message) + message = report.render() + smtp.send_message(smtp_user, report.recipient_email, message) @classmethod def write_csv(cls, report, add_headers=True): @@ -271,11 +276,6 @@ def write_csv(cls, report, add_headers=True): class EmailReport(JinjaReport): - templates = { - 'nightly_report': 'email_nightly_report.txt.j2', - 'token_expiration': 'email_token_expiration.txt.j2', - 'workflow_report': 'email_workflow_report.txt.j2', - } fields = [ 'report', 'sender_name', @@ -283,6 +283,35 @@ class EmailReport(JinjaReport): 'recipient_email', ] + def __init__(self, template_name, **kwargs): + self._template_name = template_name + super().__init__(**kwargs) + + @property + def templates(self): + return { + self._template_name: f'email_{self._template_name}.txt.j2', + } + + def date(self): + return None + + def render(self): + message = email.message.EmailMessage() + message.set_charset('utf-8') + message['Message-Id'] = email.utils.make_msgid() + date = self.date() + if isinstance(date, datetime.datetime): + message['Date'] = date + else: + message['Date'] = email.utils.formatdate(date) + message['From'] = email.headerregistry.Address( + self.sender_name, addr_spec=self.sender_email) + message['To'] = email.headerregistry.Address(addr_spec=self.recipient_email) + message['Subject'] = self.subject() + message.set_content(super().render(self._template_name)) + return message + class CommentReport(Report): diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt similarity index 83% rename from dev/archery/archery/crossbow/tests/fixtures/email-report.txt rename to dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt index c29cafd3938..5e7b8e9c67d 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt +++ b/dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt @@ -1,6 +1,11 @@ +MIME-Version: 1.0 +Message-Id: +Date: date From: Sender Reporter To: recipient@arrow.com Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit Arrow Build Report for Job ursabot-1 diff --git a/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt b/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt new file mode 100644 index 00000000000..1f8ccbf30c6 --- /dev/null +++ b/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt @@ -0,0 +1,14 @@ +MIME-Version: 1.0 +Message-Id: +Date: date +From: Sender Reporter +To: recipient@arrow.com +Subject: [CI] Arrow Crossbow Token Expiration in 2026-01-17 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: 7bit + +The Arrow Crossbow Token will expire in 7 days. + +Please generate a new Token. Send it to Apache INFRA to update the +CROSSBOW_GITHUB_TOKEN. Update it on the crossbow repository and in +the Azure pipelines. diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py index 620b4c78bbc..02012d2f1be 100644 --- a/dev/archery/archery/crossbow/tests/test_reports.py +++ b/dev/archery/archery/crossbow/tests/test_reports.py @@ -15,11 +15,12 @@ # specific language governing permissions and limitations # under the License. +import re import textwrap +from archery.crossbow.cli import (NightlyEmailReport, TokenExpirationEmailReport) from archery.crossbow.core import yaml -from archery.crossbow.reports import (ChatReport, CommentReport, EmailReport, - Report) +from archery.crossbow.reports import (ChatReport, CommentReport, Report) def test_crossbow_comment_formatter(load_fixture): @@ -71,19 +72,55 @@ def test_crossbow_chat_report_extra_message_success(load_fixture): assert report_chat.render("text") == textwrap.dedent(expected_msg) -def test_crossbow_email_report(load_fixture): - expected_msg = load_fixture('email-report.txt') +def test_crossbow_nightly_email_report(load_fixture): + expected_msg = load_fixture('nightly-email-report.txt') job = load_fixture('crossbow-job.yaml', decoder=yaml.load) report = Report(job) assert report.tasks_by_state is not None - email_report = EmailReport(report=report, sender_name="Sender Reporter", - sender_email="sender@arrow.com", - recipient_email="recipient@arrow.com") + email_report = NightlyEmailReport( + report=report, + sender_name='Sender Reporter', + sender_email='sender@arrow.com', + recipient_email='recipient@arrow.com' + ) - assert ( - email_report.render("nightly_report") == textwrap.dedent(expected_msg) + actual = str(email_report.render()) + # Normalize dynamic headers + actual = re.sub(r'(?m)^Message-Id: <.+?>', + 'Message-Id: ', + actual) + actual = re.sub(r'(?m)^Date: [^\n]+ -0000$', + 'Date: date', + actual) + assert actual == textwrap.dedent(expected_msg) + + +def test_crossbow_token_expiration_email_report(load_fixture): + expected_msg = load_fixture('token-expiration-email-report.txt') + + class TokenExpirationReport: + def __init__(self, token_expiration_date, days_left): + self.token_expiration_date = token_expiration_date + self.days_left = days_left + + report = TokenExpirationReport('2026-01-17', 7) + email_report = TokenExpirationEmailReport( + report=report, + sender_name='Sender Reporter', + sender_email='sender@arrow.com', + recipient_email='recipient@arrow.com' ) + actual = str(email_report.render()) + # Normalize dynamic headers + actual = re.sub(r'(?m)^Message-Id: <.+?>', + 'Message-Id: ', + actual) + actual = re.sub(r'(?m)^Date: [^\n]+ -0000$', + 'Date: date', + actual) + assert actual == textwrap.dedent(expected_msg) + def test_crossbow_export_report(load_fixture): job = load_fixture('crossbow-job.yaml', decoder=yaml.load) diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2 b/dev/archery/archery/templates/email_nightly_report.txt.j2 index bc040734b03..7b43d7c867e 100644 --- a/dev/archery/archery/templates/email_nightly_report.txt.j2 +++ b/dev/archery/archery/templates/email_nightly_report.txt.j2 @@ -15,13 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{ (report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"] | length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending - +-#} Arrow Build Report for Job {{ report.job.branch }} See https://s3.amazonaws.com/arrow-data/index.html for more information. @@ -58,4 +52,4 @@ Succeeded Tasks: - {{ task_name }} {{ report.task_url(task) }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%} diff --git a/dev/archery/archery/templates/email_token_expiration.txt.j2 b/dev/archery/archery/templates/email_token_expiration.txt.j2 index 54c2005e57e..340cb4a5353 100644 --- a/dev/archery/archery/templates/email_token_expiration.txt.j2 +++ b/dev/archery/archery/templates/email_token_expiration.txt.j2 @@ -15,12 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [CI] Arrow Crossbow Token Expiration in {{ report.token_expiration_date }} - +-#} The Arrow Crossbow Token will expire in {{ report.days_left }} days. -Please generate a new Token. Send it to Apache INFRA to update the CROSSBOW_GITHUB_TOKEN. -Update it on the crossbow repository and in the Azure pipelines. +Please generate a new Token. Send it to Apache INFRA to update the +CROSSBOW_GITHUB_TOKEN. Update it on the crossbow repository and in +the Azure pipelines. diff --git a/dev/archery/archery/templates/email_workflow_report.txt.j2 b/dev/archery/archery/templates/email_workflow_report.txt.j2 index 193856c1806..6668d6c67ee 100644 --- a/dev/archery/archery/templates/email_workflow_report.txt.j2 +++ b/dev/archery/archery/templates/email_workflow_report.txt.j2 @@ -15,13 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -#} -{%- if True -%} -{%- endif -%} -From: {{ sender_name }} <{{ sender_email }}> -To: {{ recipient_email }} -Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for {{ report.name }}: {{ report.failed_jobs() | length }} failed - +-#} Arrow Build Report for {{ report.name }} Workflow URL: {{ report.url }} @@ -42,4 +36,4 @@ Succeeded Jobs: - {{ job.name }} {{ job.url }} {% endfor %} -{%- endif -%} \ No newline at end of file +{%- endif -%}