From 429a154d0cb5aa2456208cba4a2f966a397601a5 Mon Sep 17 00:00:00 2001 From: Eric Lavigne Date: Sun, 8 Feb 2026 10:31:06 -0700 Subject: [PATCH 1/3] Test attach results perf --- evergreen.yml | 29 ++++++++++ generate_results_file.py | 111 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 generate_results_file.py diff --git a/evergreen.yml b/evergreen.yml index 35e9581f..ebf27d5e 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -20,6 +20,16 @@ buildvariants: tasks: - name: checkrun_test + - name: attach-results-perf + display_name: "Attach Results Performance Test" + run_on: ubuntu2204-small + expansions: + num_tests: "500" + failure_rate: "0.1" + log_lines: "20" + tasks: + - attach-results-perf-test + functions: create virtualenv: - command: shell.exec @@ -29,6 +39,11 @@ functions: echo "noop" git describe +post: + - command: attach.results + params: + file_location: src/test_results.json + tasks: - name: checkrun_test @@ -39,6 +54,20 @@ tasks: script: | echo "i am become checkrun" + # Performance test for attach.results + # Generates a large JSON file with many test results to test parallel log uploading. + - name: attach-results-perf-test + commands: + - command: subprocess.exec + params: + binary: python3 + working_dir: src + args: + - "generate_results_file.py" + - "${num_tests|500}" # Number of test results + - "${failure_rate|0.1}" # Failure rate (0.0-1.0) + - "${log_lines|20}" # Log lines per test + modules: - name: test-trigger repo: git@github.com:evergreen-ci/commit-queue-sandbox.git diff --git a/generate_results_file.py b/generate_results_file.py new file mode 100644 index 00000000..1c0a59cf --- /dev/null +++ b/generate_results_file.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +Generate a large JSON results file for performance testing of attach.results. + +Usage: + python3 generate_results_file.py [num_tests] [failure_rate] [log_lines] + +Arguments: + num_tests Number of test results to generate (default: 500) + failure_rate Fraction of tests that should fail, 0.0-1.0 (default: 0.1) + log_lines Number of lines in each test's log_raw (default: 20) + +Example: + python3 generate_results_file.py 500 0.1 20 + # Generates 500 test results, 10% failures, 20 log lines each +""" + +import json +import random +import sys +import time + + +def generate_log_content(test_name: str, num_lines: int, failed: bool) -> str: + """Generate realistic log content for a test.""" + lines = [] + lines.append(f"=== Starting test: {test_name} ===") + lines.append(f"Test configuration loaded at {time.strftime('%Y-%m-%d %H:%M:%S')}") + + for i in range(num_lines - 4): + if failed and i == num_lines - 6: + lines.append(f"ERROR: Assertion failed at line {random.randint(50, 200)}") + lines.append(f" Expected: {random.randint(1, 100)}") + lines.append(f" Actual: {random.randint(1, 100)}") + else: + log_type = random.choice(["INFO", "DEBUG", "TRACE"]) + lines.append(f"[{log_type}] Processing step {i+1}: operation completed successfully") + + status = "FAILED" if failed else "PASSED" + lines.append(f"=== Test {test_name} {status} ===") + + return "\n".join(lines) + + +def generate_test_result(test_num: int, failure_rate: float, log_lines: int) -> dict: + """Generate a single test result.""" + module_name = f"module_{test_num // 100:03d}" + test_name = f"test_{module_name}.Test{module_name.title()}Suite.test_case_{test_num:05d}" + + should_fail = random.random() < failure_rate + status = "fail" if should_fail else "pass" + + # Generate timestamps (Python time format - seconds since epoch) + start_time = 1700000000.0 + (test_num * 0.5) # Stagger start times + duration = random.uniform(0.1, 2.0) + end_time = start_time + duration + + log_content = generate_log_content(test_name, log_lines, should_fail) + + return { + "test_file": test_name, + "status": status, + "start": start_time, + "end": end_time, + "log_raw": log_content + } + + +def main(): + num_tests = int(sys.argv[1]) if len(sys.argv) > 1 else 500 + failure_rate = float(sys.argv[2]) if len(sys.argv) > 2 else 0.1 + log_lines = int(sys.argv[3]) if len(sys.argv) > 3 else 20 + + print(f"Generating results file with {num_tests} tests...") + print(f"Expected failures: ~{int(num_tests * failure_rate)}") + print(f"Log lines per test: {log_lines}") + print() + + start_time = time.time() + + results = [] + num_failures = 0 + + for test_num in range(num_tests): + result = generate_test_result(test_num, failure_rate, log_lines) + results.append(result) + if result["status"] == "fail": + num_failures += 1 + + if (test_num + 1) % 100 == 0: + print(f" Generated {test_num + 1}/{num_tests} results...") + + output = {"results": results} + + output_file = "test_results.json" + with open(output_file, "w") as f: + json.dump(output, f, indent=2) + + elapsed = time.time() - start_time + file_size = len(json.dumps(output)) / 1024 / 1024 # MB + + print() + print(f"Done! Generated {num_tests} test results in {elapsed:.2f}s") + print(f"Total failures: {num_failures}") + print(f"Output file: {output_file} ({file_size:.2f} MB)") + print() + print("File will be picked up by attach.results in post section.") + + +if __name__ == "__main__": + main() From c5938ab0147eff60cd8bd81fc5fdf92c82ff62f5 Mon Sep 17 00:00:00 2001 From: Eric Lavigne Date: Sun, 8 Feb 2026 10:51:29 -0700 Subject: [PATCH 2/3] Test go test parse results --- evergreen.yml | 28 ++++++++++ generate_gotest_files.py | 116 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 generate_gotest_files.py diff --git a/evergreen.yml b/evergreen.yml index ebf27d5e..dc80fbed 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -30,6 +30,16 @@ buildvariants: tasks: - attach-results-perf-test + - name: gotest-perf + display_name: "GoTest Parse Performance Test" + run_on: ubuntu2204-small + expansions: + num_files: "10" + tests_per_file: "100" + failure_rate: "0.1" + tasks: + - gotest-perf-test + functions: create virtualenv: - command: shell.exec @@ -43,6 +53,10 @@ post: - command: attach.results params: file_location: src/test_results.json + - command: gotest.parse_files + params: + files: + - "src/*.suite" tasks: @@ -68,6 +82,20 @@ tasks: - "${failure_rate|0.1}" # Failure rate (0.0-1.0) - "${log_lines|20}" # Log lines per test + # Performance test for gotest.parse_files + # Generates multiple .suite files with go test output to test parallel log uploading. + - name: gotest-perf-test + commands: + - command: subprocess.exec + params: + binary: python3 + working_dir: src + args: + - "generate_gotest_files.py" + - "${num_files|10}" # Number of .suite files + - "${tests_per_file|100}" # Tests per file + - "${failure_rate|0.1}" # Failure rate (0.0-1.0) + modules: - name: test-trigger repo: git@github.com:evergreen-ci/commit-queue-sandbox.git diff --git a/generate_gotest_files.py b/generate_gotest_files.py new file mode 100644 index 00000000..31655ed4 --- /dev/null +++ b/generate_gotest_files.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Generate go test output files for performance testing of gotest.parse_files. + +Usage: + python3 generate_gotest_files.py [num_files] [tests_per_file] [failure_rate] + +Arguments: + num_files Number of .suite files to generate (default: 10) + tests_per_file Number of test cases per file (default: 100) + failure_rate Fraction of tests that should fail, 0.0-1.0 (default: 0.1) + +Example: + python3 generate_gotest_files.py 10 100 0.1 + # Generates 10 files with 100 tests each (1000 total), 10% failures +""" + +import os +import random +import sys +import time + + +def generate_test_output(test_num: int, module_name: str, should_fail: bool) -> tuple[str, str]: + """Generate go test output for a single test.""" + test_name = f"Test{module_name.title()}_Case{test_num:04d}" + duration = random.uniform(0.001, 0.5) + + lines = [] + lines.append(f"=== RUN {test_name}") + + # Add some log output + for i in range(random.randint(2, 8)): + lines.append(f" {test_name}: log line {i+1}: processing step {i+1}") + + if should_fail: + lines.append(f" {test_name}: assertion failed") + lines.append(f" Expected: {random.randint(1, 100)}") + lines.append(f" Actual: {random.randint(1, 100)}") + status = "FAIL" + else: + status = "PASS" + + lines.append(f"--- {status}: {test_name} ({duration:.3f}s)") + + return "\n".join(lines), status + + +def generate_suite_file(file_num: int, tests_per_file: int, failure_rate: float, output_dir: str) -> tuple[int, int]: + """Generate a single go test .suite file.""" + module_name = f"module_{file_num:04d}" + + lines = [] + num_failures = 0 + num_passes = 0 + + for test_num in range(tests_per_file): + should_fail = random.random() < failure_rate + output, status = generate_test_output(test_num, module_name, should_fail) + lines.append(output) + if status == "FAIL": + num_failures += 1 + else: + num_passes += 1 + + # Add summary line + if num_failures > 0: + lines.append(f"FAIL") + else: + lines.append(f"PASS") + lines.append(f"ok \tgithub.com/test/{module_name}\t{random.uniform(0.5, 5.0):.3f}s") + + filename = os.path.join(output_dir, f"{module_name}.suite") + with open(filename, "w") as f: + f.write("\n".join(lines)) + + return tests_per_file, num_failures + + +def main(): + num_files = int(sys.argv[1]) if len(sys.argv) > 1 else 10 + tests_per_file = int(sys.argv[2]) if len(sys.argv) > 2 else 100 + failure_rate = float(sys.argv[3]) if len(sys.argv) > 3 else 0.1 + + output_dir = "." + + print(f"Generating {num_files} go test suite files with {tests_per_file} tests each...") + print(f"Total tests: {num_files * tests_per_file}") + print(f"Expected failures: ~{int(num_files * tests_per_file * failure_rate)}") + print() + + start_time = time.time() + + total_tests = 0 + total_failures = 0 + + for file_num in range(num_files): + tests, failures = generate_suite_file(file_num, tests_per_file, failure_rate, output_dir) + total_tests += tests + total_failures += failures + + if (file_num + 1) % 5 == 0: + print(f" Generated {file_num + 1}/{num_files} files...") + + elapsed = time.time() - start_time + + print() + print(f"Done! Generated {num_files} files in {elapsed:.2f}s") + print(f"Total tests: {total_tests}") + print(f"Total failures: {total_failures}") + print() + print("Files will be picked up by gotest.parse_files in post section.") + + +if __name__ == "__main__": + main() From 4fd22397f87c4ed7a3253358461d0d9ab810111d Mon Sep 17 00:00:00 2001 From: Eric Lavigne Date: Sun, 8 Feb 2026 11:11:59 -0700 Subject: [PATCH 3/3] Test upload traces perf --- evergreen.yml | 26 +++++++ generate_trace_files.py | 153 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 generate_trace_files.py diff --git a/evergreen.yml b/evergreen.yml index dc80fbed..58ebaf82 100644 --- a/evergreen.yml +++ b/evergreen.yml @@ -40,6 +40,15 @@ buildvariants: tasks: - gotest-perf-test + - name: upload-traces-perf + display_name: "Upload Traces Performance Test" + run_on: ubuntu2204-small + expansions: + num_files: "74" + spans_per_file: "5000" + tasks: + - upload-traces-perf-test + functions: create virtualenv: - command: shell.exec @@ -96,6 +105,23 @@ tasks: - "${tests_per_file|100}" # Tests per file - "${failure_rate|0.1}" # Failure rate (0.0-1.0) + # Performance test for upload-traces + # Generates OTel trace files to test parallel trace uploading. + - name: upload-traces-perf-test + commands: + - command: shell.exec + params: + script: | + echo "OTel collector endpoint: '${otel_collector_endpoint}'" + - command: subprocess.exec + params: + binary: python3 + working_dir: src + args: + - "generate_trace_files.py" + - "${num_files|10}" # Number of trace files + - "${spans_per_file|100}" # Spans per file + modules: - name: test-trigger repo: git@github.com:evergreen-ci/commit-queue-sandbox.git diff --git a/generate_trace_files.py b/generate_trace_files.py new file mode 100644 index 00000000..93c9d91e --- /dev/null +++ b/generate_trace_files.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +""" +Generate OTel trace files for performance testing of upload-traces. + +Usage: + python3 generate_trace_files.py [num_files] [spans_per_file] + +Arguments: + num_files Number of trace files to generate (default: 10) + spans_per_file Number of spans per file (default: 100) + +Example: + python3 generate_trace_files.py 10 100 + # Generates 10 trace files with 100 spans each +""" + +import json +import os +import random +import sys +import time + + +def generate_trace_id(): + """Generate a random 32-character hex trace ID.""" + return ''.join(random.choices('0123456789abcdef', k=32)) + + +def generate_span_id(): + """Generate a random 16-character hex span ID.""" + return ''.join(random.choices('0123456789abcdef', k=16)) + + +def generate_span(trace_id: str, parent_span_id: str, span_num: int, start_time_ns: int) -> dict: + """Generate a single span.""" + span_id = generate_span_id() + duration_ns = random.randint(1000000, 100000000) # 1ms to 100ms + + return { + "traceId": trace_id, + "spanId": span_id, + "parentSpanId": parent_span_id, + "name": f"operation_{span_num:04d}", + "kind": random.randint(1, 5), + "startTimeUnixNano": str(start_time_ns), + "endTimeUnixNano": str(start_time_ns + duration_ns), + "attributes": [ + {"key": "db.system", "value": {"stringValue": "mongodb"}}, + {"key": "db.operation", "value": {"stringValue": random.choice(["find", "insert", "update", "delete"])}}, + {"key": "db.name", "value": {"stringValue": "test_db"}}, + {"key": "span.num", "value": {"intValue": str(span_num)}}, + ], + "status": {} + } + + +def generate_trace_data(num_spans: int) -> dict: + """Generate a complete trace data object with multiple spans.""" + trace_id = generate_trace_id() + base_time_ns = int(time.time() * 1e9) + + spans = [] + parent_span_id = "" + + for i in range(num_spans): + span = generate_span(trace_id, parent_span_id, i, base_time_ns + (i * 1000000)) + spans.append(span) + # Randomly decide if next span is a child or sibling + if random.random() < 0.3: + parent_span_id = span["spanId"] + elif random.random() < 0.5: + parent_span_id = "" + + return { + "resourceSpans": [{ + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "test-service"}}, + {"key": "service.version", "value": {"stringValue": "1.0.0"}}, + {"key": "host.name", "value": {"stringValue": "test-host"}}, + ] + }, + "scopeSpans": [{ + "scope": { + "name": "test-tracer" + }, + "spans": spans + }] + }] + } + + +def generate_trace_file(file_num: int, spans_per_file: int, output_dir: str) -> int: + """Generate a single trace file with multiple trace data lines.""" + lines = [] + + # Generate multiple trace data objects per file (simulating collector output) + # Real-world data shows ~275 spans per JSON line, so we target that density + spans_per_object = 275 + num_trace_objects = max(1, spans_per_file // spans_per_object) + + actual_spans = 0 + for _ in range(num_trace_objects): + trace_data = generate_trace_data(spans_per_object) + lines.append(json.dumps(trace_data)) + actual_spans += spans_per_object + + filename = os.path.join(output_dir, f"traces_{file_num:04d}.jsonl") + with open(filename, "w") as f: + f.write("\n".join(lines) + "\n") + + return actual_spans + + +def main(): + num_files = int(sys.argv[1]) if len(sys.argv) > 1 else 10 + spans_per_file = int(sys.argv[2]) if len(sys.argv) > 2 else 100 + + # Output to OTelTraces directory (where the agent looks for traces) + # The agent looks at /build/OTelTraces, not inside src/ + output_dir = "../build/OTelTraces" + os.makedirs(output_dir, exist_ok=True) + + print(f"Generating {num_files} trace files with ~{spans_per_file} spans each...") + print(f"Total spans: ~{num_files * spans_per_file}") + print(f"Output directory: {output_dir}") + print() + + start_time = time.time() + + total_spans = 0 + for file_num in range(num_files): + spans = generate_trace_file(file_num, spans_per_file, output_dir) + total_spans += spans + + if (file_num + 1) % 5 == 0: + print(f" Generated {file_num + 1}/{num_files} files...") + + elapsed = time.time() - start_time + + print() + print(f"Done! Generated {num_files} files in {elapsed:.2f}s") + print(f"Total spans: {total_spans}") + + # Verify files were created + actual_files = [f for f in os.listdir(output_dir) if f.endswith('.jsonl')] + print(f"Files in output directory: {len(actual_files)}") + print() + print("Files will be picked up by upload-traces handler automatically.") + + +if __name__ == "__main__": + main()