Skip to content

Commit 3072eb1

Browse files
gdyuldingdyuldin
committed
Add repeat-scope parameter
This parameter allows to change test execution order. For example, with `--repeat-scope session` all tests executes first time, next all tests executes second time and etc.
1 parent bb6b7ac commit 3072eb1

File tree

3 files changed

+157
-1
lines changed

3 files changed

+157
-1
lines changed

‎README.rst‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,19 @@ can use the :code:`@pytest.mark.repeat(count)` decorator:
6060
def test_repeat_decorator():
6161
pass
6262
63+
If you want to override default tests executions order, you can use :code:`--repeat-scope`
64+
command line option with one of the next values: :code:`session`, :code:`module`, :code:`class` or :code:`function` (default).
65+
It behaves like a scope of the pytest fixture.
66+
67+
:code:`function` (default) scope repeats each test :code:`count` or :code:`repeat` times before executing next test.
68+
:code:`session` scope repeats whole tests session, i.e. all collected tests executed once, then all such tests executed again and etc.
69+
:code:`class` and :code:`module` behaves similar :code:`session` , but repeating set of tests is a tests from class or module, not all collected tests.
70+
6371
Repeating a test until failure
6472
------------------------------
6573

6674
If you are trying to diagnose an intermittent failure, it can be useful to run the same
67-
test over and over again until it fails. You can use py.test's :code:`-x` option in
75+
test over and over again until it fails. You can use pytest's :code:`-x` option in
6876
conjunction with pytest-repeat to force the test runner to stop at the first failure.
6977
For example:
7078

‎pytest_repeat.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@ def pytest_addoption(parser):
1515
type=int,
1616
help='Number of times to repeat each test')
1717

18+
parser.addoption(
19+
'--repeat-scope',
20+
action='store',
21+
default='function',
22+
type=str,
23+
choices=('function', 'class', 'module', 'session'),
24+
help='Scope for repeating tests')
25+
1826

1927
def pytest_configure(config):
2028
config.addinivalue_line(
@@ -50,9 +58,11 @@ def pytest_generate_tests(metafunc):
5058
def make_progress_id(i, n=count):
5159
return '{0}/{1}'.format(i + 1, n)
5260

61+
scope = metafunc.config.option.repeat_scope
5362
metafunc.parametrize(
5463
'__pytest_repeat_step_number',
5564
range(count),
5665
indirect=True,
5766
ids=make_progress_id,
67+
scope=scope
5868
)

‎test_repeat.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
# License, v. 2.0. If a copy of the MPL was not distributed with this
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

5+
import pytest
6+
57
pytest_plugins = "pytester",
68

79

@@ -111,3 +113,139 @@ def test_this(self):
111113
'*test_unittest_test.py::ClassStyleTest::test_this PASSED*',
112114
'*1 passed*',
113115
])
116+
117+
@pytest.mark.parametrize(['scope', 'lines'], [
118+
('session', [
119+
'*test_1.py::test_repeat1[[]1/2[]] PASSED*',
120+
'*test_1.py::test_repeat2[[]1/2[]] PASSED*',
121+
'*test_2.py::test_repeat3[[]1/2[]] PASSED*',
122+
'*test_2.py::test_repeat4[[]1/2[]] PASSED*',
123+
'*test_3.py::TestRepeat1::test_repeat5[[]1/2[]] PASSED*',
124+
'*test_3.py::TestRepeat1::test_repeat6[[]1/2[]] PASSED*',
125+
'*test_3.py::TestRepeat2::test_repeat7[[]1/2[]] PASSED*',
126+
'*test_3.py::TestRepeat2::test_repeat8[[]1/2[]] PASSED*',
127+
'*test_1.py::test_repeat1[[]2/2[]] PASSED*',
128+
'*test_1.py::test_repeat2[[]2/2[]] PASSED*',
129+
'*test_2.py::test_repeat3[[]2/2[]] PASSED*',
130+
'*test_2.py::test_repeat4[[]2/2[]] PASSED*',
131+
'*test_3.py::TestRepeat1::test_repeat5[[]2/2[]] PASSED*',
132+
'*test_3.py::TestRepeat1::test_repeat6[[]2/2[]] PASSED*',
133+
'*test_3.py::TestRepeat2::test_repeat7[[]2/2[]] PASSED*',
134+
'*test_3.py::TestRepeat2::test_repeat8[[]2/2[]] PASSED*',
135+
'*16 passed*',
136+
]),
137+
('module', [
138+
'*test_1.py::test_repeat1[[]1/2[]] PASSED*',
139+
'*test_1.py::test_repeat2[[]1/2[]] PASSED*',
140+
'*test_1.py::test_repeat1[[]2/2[]] PASSED*',
141+
'*test_1.py::test_repeat2[[]2/2[]] PASSED*',
142+
'*test_2.py::test_repeat3[[]1/2[]] PASSED*',
143+
'*test_2.py::test_repeat4[[]1/2[]] PASSED*',
144+
'*test_2.py::test_repeat3[[]2/2[]] PASSED*',
145+
'*test_2.py::test_repeat4[[]2/2[]] PASSED*',
146+
'*test_3.py::TestRepeat1::test_repeat5[[]1/2[]] PASSED*',
147+
'*test_3.py::TestRepeat1::test_repeat6[[]1/2[]] PASSED*',
148+
'*test_3.py::TestRepeat2::test_repeat7[[]1/2[]] PASSED*',
149+
'*test_3.py::TestRepeat2::test_repeat8[[]1/2[]] PASSED*',
150+
'*test_3.py::TestRepeat1::test_repeat5[[]2/2[]] PASSED*',
151+
'*test_3.py::TestRepeat1::test_repeat6[[]2/2[]] PASSED*',
152+
'*test_3.py::TestRepeat2::test_repeat7[[]2/2[]] PASSED*',
153+
'*test_3.py::TestRepeat2::test_repeat8[[]2/2[]] PASSED*',
154+
'*16 passed*',
155+
]),
156+
('class', [
157+
'*test_1.py::test_repeat1[[]1/2[]] PASSED*',
158+
'*test_1.py::test_repeat2[[]1/2[]] PASSED*',
159+
'*test_1.py::test_repeat1[[]2/2[]] PASSED*',
160+
'*test_1.py::test_repeat2[[]2/2[]] PASSED*',
161+
'*test_2.py::test_repeat3[[]1/2[]] PASSED*',
162+
'*test_2.py::test_repeat4[[]1/2[]] PASSED*',
163+
'*test_2.py::test_repeat3[[]2/2[]] PASSED*',
164+
'*test_2.py::test_repeat4[[]2/2[]] PASSED*',
165+
'*test_3.py::TestRepeat1::test_repeat5[[]1/2[]] PASSED*',
166+
'*test_3.py::TestRepeat1::test_repeat6[[]1/2[]] PASSED*',
167+
'*test_3.py::TestRepeat1::test_repeat5[[]2/2[]] PASSED*',
168+
'*test_3.py::TestRepeat1::test_repeat6[[]2/2[]] PASSED*',
169+
'*test_3.py::TestRepeat2::test_repeat7[[]1/2[]] PASSED*',
170+
'*test_3.py::TestRepeat2::test_repeat8[[]1/2[]] PASSED*',
171+
'*test_3.py::TestRepeat2::test_repeat7[[]2/2[]] PASSED*',
172+
'*test_3.py::TestRepeat2::test_repeat8[[]2/2[]] PASSED*',
173+
'*16 passed*',
174+
]),
175+
('function', [
176+
'*test_1.py::test_repeat1[[]1/2[]] PASSED*',
177+
'*test_1.py::test_repeat1[[]2/2[]] PASSED*',
178+
'*test_1.py::test_repeat2[[]1/2[]] PASSED*',
179+
'*test_1.py::test_repeat2[[]2/2[]] PASSED*',
180+
'*test_2.py::test_repeat3[[]1/2[]] PASSED*',
181+
'*test_2.py::test_repeat3[[]2/2[]] PASSED*',
182+
'*test_2.py::test_repeat4[[]1/2[]] PASSED*',
183+
'*test_2.py::test_repeat4[[]2/2[]] PASSED*',
184+
'*test_3.py::TestRepeat1::test_repeat5[[]1/2[]] PASSED*',
185+
'*test_3.py::TestRepeat1::test_repeat5[[]2/2[]] PASSED*',
186+
'*test_3.py::TestRepeat1::test_repeat6[[]1/2[]] PASSED*',
187+
'*test_3.py::TestRepeat1::test_repeat6[[]2/2[]] PASSED*',
188+
'*test_3.py::TestRepeat2::test_repeat7[[]1/2[]] PASSED*',
189+
'*test_3.py::TestRepeat2::test_repeat7[[]2/2[]] PASSED*',
190+
'*test_3.py::TestRepeat2::test_repeat8[[]1/2[]] PASSED*',
191+
'*test_3.py::TestRepeat2::test_repeat8[[]2/2[]] PASSED*',
192+
'*16 passed*',
193+
]),
194+
])
195+
def test_scope(self, testdir, scope, lines):
196+
testdir.makepyfile(test_1="""
197+
def test_repeat1():
198+
pass
199+
200+
def test_repeat2():
201+
pass
202+
""")
203+
testdir.makepyfile(test_2="""
204+
def test_repeat3():
205+
pass
206+
207+
def test_repeat4():
208+
pass
209+
""")
210+
testdir.makepyfile(test_3="""
211+
class TestRepeat1(object):
212+
def test_repeat5(self):
213+
pass
214+
def test_repeat6(self):
215+
pass
216+
class TestRepeat2(object):
217+
def test_repeat7(self):
218+
pass
219+
def test_repeat8(self):
220+
pass
221+
""")
222+
result = testdir.runpytest('-v', '--count', '2', '--repeat-scope',
223+
scope)
224+
result.stdout.fnmatch_lines(lines)
225+
assert result.ret == 0
226+
227+
def test_omitted_scope(self, testdir):
228+
testdir.makepyfile("""
229+
def test_repeat1():
230+
pass
231+
232+
def test_repeat2():
233+
pass
234+
""")
235+
result = testdir.runpytest('-v', '--count', '2')
236+
result.stdout.fnmatch_lines([
237+
'*test_omitted_scope.py::test_repeat1[[]1/2[]] PASSED*',
238+
'*test_omitted_scope.py::test_repeat1[[]2/2[]] PASSED*',
239+
'*test_omitted_scope.py::test_repeat2[[]1/2[]] PASSED*',
240+
'*test_omitted_scope.py::test_repeat2[[]2/2[]] PASSED*',
241+
'*4 passed*',
242+
])
243+
assert result.ret == 0
244+
245+
def test_invalid_scope(self, testdir):
246+
testdir.makepyfile("""
247+
def test_repeat():
248+
pass
249+
""")
250+
result = testdir.runpytest('--count', '2', '--repeat-scope', 'a')
251+
assert result.ret == 2

0 commit comments

Comments
 (0)