Add --tag-filters option

Author: nprizal

src/buildkite_test_collector/pytest_plugin/__init__.py

@@ -89,3 +89,10 @@ def pytest_addoption(parser):
         dest="mergejson",
         help='merge json output with existing file, if it exists'
     )
+    group.addoption(
+        '--tag-filters',
+        default=None,
+        action='store',
+        dest="tag_filters",
+        help='filter tests by execution_tag with `key:value`, e.g. `--tag-filters color:red`'
+    )

src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py

@@ -17,6 +17,17 @@ def __init__(self, payload):
         self.in_flight = {}
         self.spans = {}
 
+    def pytest_collection_modifyitems(self, config, items):
+        """pytest_collection_modifyitems hook callback to filter tests by execution_tag markers"""
+        tag_filter = config.getoption("tag_filters")
+        if not tag_filter:
+            return
+
+        filtered_items, unfiltered_items = self._filter_tests_by_tag(items, tag_filter)
+
+        config.hook.pytest_deselected(items=unfiltered_items)
+        items[:] = filtered_items
+
     def pytest_runtest_logstart(self, nodeid, location):
         """pytest_runtest_logstart hook callback"""
         logger.debug('hook=pytest_runtest_logstart nodeid=%s', nodeid)
@@ -144,3 +155,32 @@ def save_payload_as_json(self, path, merge=False):
 
         with open(path, "w", encoding="utf-8") as f:
             json.dump(data, f)
+
+    def _filter_tests_by_tag(self, items, tag_filter):
+        """
+        Filters tests based on the tag_filter option.
+        Supports filtering by a single tag in the format key:value.
+        Only equality comparison is supported.
+        Returns a tuple of (filtered_items, unfiltered_items).
+        """
+        key, _, value = tag_filter.partition(":")
+
+        filtered_items = []
+        unfiltered_items = []
+        for item in items:
+            # Extract all execution_tag markers and store them in a dict
+            tags = {}
+            markers = item.iter_markers("execution_tag")
+            for tag_marker in markers:
+                # Ensure the marker has exactly two arguments: key and value
+                if len(tag_marker.args) != 2:
+                    continue
+
+                tags[tag_marker.args[0]] = tag_marker.args[1]
+
+            if tags.get(key) == value:
+                filtered_items.append(item)
+            else:
+                unfiltered_items.append(item)
+
+        return filtered_items, unfiltered_items

tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py

@@ -0,0 +1,29 @@
+# This is sample test file used for integration testing of execution_tag marker
+import pytest
+
+
+@pytest.mark.execution_tag("color", "red")
+@pytest.mark.execution_tag("size", "medium")
+def test_apple():
+    assert True
+
+@pytest.mark.execution_tag("color", "orange")
+@pytest.mark.execution_tag("size", "medium")
+def test_orange():
+    assert True
+
+@pytest.mark.execution_tag("color", "yellow")
+@pytest.mark.execution_tag("size", "large")
+def test_banana():
+    assert True
+
+@pytest.mark.execution_tag("color", "purple")
+@pytest.mark.execution_tag("size", "small")
+def test_grape():
+    assert True
+
+@pytest.mark.execution_tag("color", "red")
+@pytest.mark.execution_tag("size", "small")
+def test_strawberry():
+    assert True
+

tests/buildkite_test_collector/test_integration.py

@@ -1,13 +1,13 @@
 import json
 import os
 import pytest
+import subprocess
+import sys
+    
 from pathlib import Path
 
 def test_add_tag_to_execution_data(tmp_path, fake_env):
     """Verify that tags added via the execution_tag marker are correctly captured in the test data."""
-    import subprocess
-    import sys
-    
     test_file = Path(__file__).parent / "data" / "test_sample_execution_tag.py"
     json_output_file = tmp_path / "test_results.json"
 
@@ -41,3 +41,70 @@ def test_add_tag_to_execution_data(tmp_path, fake_env):
 
     no_tag_test = tests_by_name["test_without_tags"]
     assert "tags" not in no_tag_test or no_tag_test.get("tags") == {}
+
+class TestTagFiltering:
+    import subprocess
+    import sys
+
+    def test_filter_by_single_tag(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "color:red",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        assert "2/5 tests collected" in result.stdout, "collect count mismatch"
+
+        # Parse the output to find collected tests
+        lines = result.stdout.strip().splitlines()
+        collected_tests = []
+        for line in lines:
+            if "::" in line:
+                collected_tests.append(line)
+
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" not in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" not in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" not in collected_tests
+
+    def test_wrong_filter_format(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q", "--tag-filters", "foobar",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        assert "no tests collected" in result.stdout
+
+    def test_no_filter(self,tmp_path, fake_env):
+        test_file = Path(__file__).parent / "data" / "test_sample_execution_tag_filter.py"
+        
+        cmd = [
+            sys.executable, "-m", "pytest", "--co", "-q",
+            str(test_file),
+        ]
+        
+        # Run pytest in a subprocess
+        result = subprocess.run(cmd, cwd=str(tmp_path), capture_output=True, text=True)
+
+        # Parse the output to find collected tests
+        lines = result.stdout.strip().splitlines()
+        collected_tests = []
+        for line in lines:
+            if "::" in line:
+                collected_tests.append(line)
+
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_apple" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_strawberry" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_orange" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_banana" in collected_tests
+        assert "tests/buildkite_test_collector/data/test_sample_execution_tag_filter.py::test_grape" in collected_tests