Merge pull request #83 from buildkite/te-5135-add-custom-option-to-filter-tests-by-execution-tag-in-pytest

Add `--tag-filters` option to filter tests using `execution_tag` marker

Author: gchan

src/buildkite_test_collector/pytest_plugin/__init__.py

@@ -41,7 +41,7 @@ def pytest_unconfigure(config):
 
     if plugin:
         api = API(os.environ)
-        numprocesses = config.getoption("numprocesses")
+        numprocesses = config.getoption("numprocesses", None)
         xdist_enabled = (
             config.pluginmanager.getplugin("xdist") is not None
             and numprocesses is not None
@@ -89,3 +89,10 @@ def pytest_addoption(parser):
         dest="mergejson",
         help='merge json output with existing file, if it exists'
     )
+    group.addoption(
+        '--tag-filters',
+        default=None,
+        action='store',
+        dest="tag_filters",
+        help='filter tests by execution_tag with `key:value`, e.g. `--tag-filters color:red`'
+    )

src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py

@@ -17,6 +17,17 @@ def __init__(self, payload):
         self.in_flight = {}
         self.spans = {}
 
+    def pytest_collection_modifyitems(self, config, items):
+        """pytest_collection_modifyitems hook callback to filter tests by execution_tag markers"""
+        tag_filter = config.getoption("tag_filters")
+        if not tag_filter:
+            return
+
+        filtered_items, unfiltered_items = self._filter_tests_by_tag(items, tag_filter)
+
+        config.hook.pytest_deselected(items=unfiltered_items)
+        items[:] = filtered_items
+
     def pytest_runtest_logstart(self, nodeid, location):
         """pytest_runtest_logstart hook callback"""
         logger.debug('hook=pytest_runtest_logstart nodeid=%s', nodeid)
@@ -144,3 +155,32 @@ def save_payload_as_json(self, path, merge=False):
 
         with open(path, "w", encoding="utf-8") as f:
             json.dump(data, f)
+
+    def _filter_tests_by_tag(self, items, tag_filter):
+        """
+        Filters tests based on the tag_filter option.
+        Supports filtering by a single tag in the format key:value.
+        Only equality comparison is supported.
+        Returns a tuple of (filtered_items, unfiltered_items).
+        """
+        key, _, value = tag_filter.partition(":")
+
+        filtered_items = []
+        unfiltered_items = []
+        for item in items:
+            # Extract all execution_tag markers and store them in a dict
+            tags = {}
+            markers = item.iter_markers("execution_tag")
+            for tag_marker in markers:
+                # Ensure the marker has exactly two arguments: key and value
+                if len(tag_marker.args) != 2:
+                    continue
+
+                tags[tag_marker.args[0]] = tag_marker.args[1]
+
+            if tags.get(key) == value:
+                filtered_items.append(item)
+            else:
+                unfiltered_items.append(item)
+
+        return filtered_items, unfiltered_items

tests/buildkite_test_collector/data/test_sample_execution_tag.py

@@ -0,0 +1,15 @@
+# This is sample test file used for integration testing of execution_tag marker
+import pytest
+
+
+@pytest.mark.execution_tag("language.version", "3.12")
+@pytest.mark.execution_tag("team", "backend") 
+def test_with_multiple_tags():
+    assert True
+
+@pytest.mark.execution_tag("team", "frontend")
+def test_with_single_tag():
+    assert True
+
+def test_without_tags():
+    assert True

tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py

@@ -0,0 +1,29 @@
+# This is sample test file used for integration testing of execution_tag marker
+import pytest
+
+
+@pytest.mark.execution_tag("color", "red")
+@pytest.mark.execution_tag("size", "medium")
+def test_apple():
+    assert True
+
+@pytest.mark.execution_tag("color", "orange")
+@pytest.mark.execution_tag("size", "medium")
+def test_orange():
+    assert True
+
+@pytest.mark.execution_tag("color", "yellow")
+@pytest.mark.execution_tag("size", "large")
+def test_banana():
+    assert True
+
+@pytest.mark.execution_tag("color", "purple")
+@pytest.mark.execution_tag("size", "small")
+def test_grape():
+    assert True
+
+@pytest.mark.execution_tag("color", "red")
+@pytest.mark.execution_tag("size", "small")
+def test_strawberry():
+    assert True
+

tests/buildkite_test_collector/test_integration.py

@@ -0,0 +1,114 @@
+import json
+import os
+import pytest
+import subprocess
+import sys
+    
+from pathlib import Path
+
+def test_add_tag_to_execution_data(tmp_path, fake_env):
+    """Verify that tags added via the execution_tag marker are correctly captured in the test data."""
+    test_file = Path(__file__).parent / "data" / "test_sample_execution_tag.py"
+    json_output_file = tmp_path / "test_results.json"
+    
+    # Run pytest with our plugin on the test file
+    cmd = [
+        sys.executable, "-m", "pytest", 
+        str(test_file),
+        f"--json={json_output_file}",
+    ]
+    
+    # Run pytest in a subprocess
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        pytest.fail("pytest run failed, see output above")
+
+    assert json_output_file.exists(), "JSON output file was not created"
+
+    with open(json_output_file, 'r') as f:
+        test_results = json.load(f)
+    
+    tests_by_name = {test["name"]: test for test in test_results}
+    
+    multi_tag_test = tests_by_name["test_with_multiple_tags"]
+    assert "tags" in multi_tag_test
+    assert multi_tag_test["tags"] == {
+        "language.version": "3.12",
+        "team": "backend"
+    }
+    
+    single_tag_test = tests_by_name["test_with_single_tag"] 
+    assert "tags" in single_tag_test
+    assert single_tag_test["tags"] == {"team": "frontend"}
+    
+    no_tag_test = tests_by_name["test_without_tags"]
+    assert "tags" not in no_tag_test or no_tag_test.get("tags") == {}
+
+class TestTagFiltering:
+    import subprocess
+    import sys
+
+    def test_filter_by_single_tag(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "color:red",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        assert "2/5 tests collected" in result.stdout, "collect count mismatch"
+
+        # Parse the output to find collected tests
+        lines = result.stdout.strip().splitlines()
+        collected_tests = []
+        for line in lines:
+            if "::" in line:
+                collected_tests.append(line)
+
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" not in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" not in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" not in collected_tests
+
+    def test_wrong_filter_format(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "foobar",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        assert "no tests collected" in result.stdout
+
+    def test_no_filter(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        # Parse the output to find collected tests
+        lines = result.stdout.strip().splitlines()
+        collected_tests = []
+        for line in lines:
+            if "::" in line:
+                collected_tests.append(line)
+
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" in collected_tests