Dynamic contexts

.travis.yml

@@ -81,6 +81,10 @@ jobs:
       python: '3.6'
     - env: TOXENV=py36-pytest51-xdist29-coverage50
       python: '3.6'
+    - env: TOXENV=py36-pytest52-xdist29-coverage45
+      python: '3.6'
+    - env: TOXENV=py36-pytest52-xdist29-coverage50
+      python: '3.6'
     - env: TOXENV=py37-pytest46-xdist29-coverage45
       python: '3.7'
     - env: TOXENV=py37-pytest46-xdist29-coverage50
@@ -89,6 +93,10 @@ jobs:
       python: '3.7'
     - env: TOXENV=py37-pytest51-xdist29-coverage50
       python: '3.7'
+    - env: TOXENV=py37-pytest52-xdist29-coverage45
+      python: '3.7'
+    - env: TOXENV=py37-pytest52-xdist29-coverage50
+      python: '3.7'
     - env: TOXENV=py38-pytest46-xdist29-coverage45
       python: '3.8-dev'
     - env: TOXENV=py38-pytest46-xdist29-coverage50
@@ -97,6 +105,10 @@ jobs:
       python: '3.8-dev'
     - env: TOXENV=py38-pytest51-xdist29-coverage50
       python: '3.8-dev'
+    - env: TOXENV=py38-pytest52-xdist29-coverage45
+      python: '3.8-dev'
+    - env: TOXENV=py38-pytest52-xdist29-coverage50
+      python: '3.8-dev'
     - env: TOXENV=pypy3-pytest46-xdist29-coverage45
       python: 'pypy3'
     - env: TOXENV=pypy3-pytest46-xdist29-coverage50
@@ -105,6 +117,10 @@ jobs:
       python: 'pypy3'
     - env: TOXENV=pypy3-pytest51-xdist29-coverage50
       python: 'pypy3'
+    - env: TOXENV=pypy3-pytest52-xdist29-coverage45
+      python: 'pypy3'
+    - env: TOXENV=pypy3-pytest52-xdist29-coverage50
+      python: 'pypy3'
 
     - stage: examples
       python: '3.6'

AUTHORS.rst

@@ -30,3 +30,4 @@ Authors
 * Семён Марьясин - https://github.com./MarSoft
 * Alexander Shadchin - https://github.com./shadchin
 * Thomas Grainger - https://graingert.co.uk
+* Ned Batchelder - https://nedbatchelder.com

CHANGELOG.rst

@@ -4,6 +4,8 @@ Changelog
 2.7.2.dev0 (unreleased)
 -----------------------
 
+* Added --cov-context option for setting the coverage.py dynamic context for
+  each test.
 * Match pytest-xdist master/worker terminology.
   Contributed in `#321 <https://github.com./pytest-dev/pytest-cov/pull/321>`_
 

appveyor.yml

@@ -7,8 +7,8 @@ environment:
     - TOXENV: 'py27-pytest310-xdist27-coverage45,py27-pytest46-xdist27-coverage45,py27-pytest310-xdist27-coverage50,py27-pytest46-xdist27-coverage50'
     - TOXENV: 'py34-pytest310-xdist27-coverage45,py34-pytest46-xdist27-coverage45'
     - TOXENV: 'py35-pytest310-xdist27-coverage45,py35-pytest46-xdist27-coverage45,py35-pytest310-xdist27-coverage50,py35-pytest46-xdist27-coverage50'
-    - TOXENV: 'py36-pytest310-xdist27-coverage45,py36-pytest46-xdist27-coverage45,py36-pytest310-xdist27-coverage50,py36-pytest46-xdist27-coverage50,py36-pytest46-xdist29-coverage45,py36-pytest46-xdist29-coverage50,py36-pytest51-xdist29-coverage45,py36-pytest51-xdist29-coverage50'
-    - TOXENV: 'py37-pytest310-xdist27-coverage45,py37-pytest46-xdist27-coverage45,py37-pytest310-xdist27-coverage50,py37-pytest46-xdist27-coverage50,py37-pytest46-xdist29-coverage45,py37-pytest46-xdist29-coverage50,py37-pytest51-xdist29-coverage45,py37-pytest51-xdist29-coverage50'
+    - TOXENV: 'py36-pytest310-xdist27-coverage45,py36-pytest46-xdist27-coverage45,py36-pytest310-xdist27-coverage50,py36-pytest46-xdist27-coverage50,py36-pytest46-xdist29-coverage45,py36-pytest46-xdist29-coverage50,py36-pytest51-xdist29-coverage45,py36-pytest51-xdist29-coverage50,py36-pytest52-xdist29-coverage45,py36-pytest52-xdist29-coverage50'
+    - TOXENV: 'py37-pytest310-xdist27-coverage45,py37-pytest46-xdist27-coverage45,py37-pytest310-xdist27-coverage50,py37-pytest46-xdist27-coverage50,py37-pytest46-xdist29-coverage45,py37-pytest46-xdist29-coverage50,py37-pytest51-xdist29-coverage45,py37-pytest51-xdist29-coverage50,py37-pytest52-xdist29-coverage45,py37-pytest52-xdist29-coverage50'
     - TOXENV: 'pypy-pytest310-xdist27-coverage45,pypy-pytest46-xdist27-coverage45,pypy-pytest310-xdist27-coverage50,pypy-pytest46-xdist27-coverage50'
 
 init:

docs/config.rst

@@ -63,3 +63,4 @@ The complete list of command line options is:
   --cov-append          Do not delete coverage but append to current. Default:
                         False
   --cov-branch          Enable branch coverage.
+  --cov-context         Choose the method for setting the dynamic context.

docs/contexts.rst

@@ -0,0 +1,18 @@
+========
+Contexts
+========
+
+Coverage.py 5.0 can record separate coverage data for different contexts during
+one run of a test suite.  Pytest-cov can use this feature to record coverage
+data for each test individually, with the ``--cov-context=test`` option.
+
+The context name recorded in the coverage.py database is the pytest test id,
+and the phase of execution, one of "setup", "run", or "teardown".  These two
+are separated with a pipe symbol.  You might see contexts like::
+
+    test_functions.py::test_addition|run
+    test_fancy.py::test_parametrized[1-101]|setup
+    test_oldschool.py::RegressionTests::test_error|run
+
+Note that parameterized tests include the values of the parameters in the test
+id, and each set of parameter values is recorded as a separate test.

docs/index.rst

@@ -12,6 +12,7 @@ Contents:
     debuggers
     xdist
     subprocess-support
+    contexts
     tox
     plugins
     markers-fixtures

setup.py

@@ -1,23 +1,24 @@
 #!/usr/bin/env python
 # -*- encoding: utf-8 -*-
-from __future__ import absolute_import, print_function
+from __future__ import absolute_import
+from __future__ import print_function
 
 import io
-from itertools import chain
 import re
+from distutils.command.build import build
 from glob import glob
+from itertools import chain
 from os.path import basename
 from os.path import dirname
 from os.path import join
 from os.path import splitext
-from distutils.command.build import build
 
 from setuptools import Command
 from setuptools import find_packages
 from setuptools import setup
 from setuptools.command.develop import develop
-from setuptools.command.install_lib import install_lib
 from setuptools.command.easy_install import easy_install
+from setuptools.command.install_lib import install_lib
 
 
 def read(*names, **kwargs):

src/pytest_cov/engine.py

@@ -1,5 +1,4 @@
 """Coverage controllers for use by pytest-cov and nose-cov."""
-
 import contextlib
 import copy
 import os
@@ -10,8 +9,10 @@
 import coverage
 from coverage.data import CoverageData
 
+from .compat import StringIO
+from .compat import workerinput
+from .compat import workeroutput
 from .embed import cleanup
-from .compat import StringIO, workeroutput, workerinput
 
 
 class _NullFile(object):
@@ -130,8 +131,7 @@ def summary(self, stream):
                 'file': stream,
             }
             skip_covered = isinstance(self.cov_report, dict) and 'skip-covered' in self.cov_report.values()
-            if hasattr(coverage, 'version_info') and coverage.version_info[0] >= 4:
-                options.update({'skip_covered': skip_covered or None})
+            options.update({'skip_covered': skip_covered or None})
             with _backup(self.cov, "config"):
                 total = self.cov.report(**options)
 

src/pytest_cov/plugin.py

@@ -3,6 +3,7 @@
 import os
 import warnings
 
+import coverage
 import pytest
 from coverage.misc import CoverageException
 
@@ -48,6 +49,14 @@ def validate_fail_under(num_str):
         return float(num_str)
 
 
+def validate_context(arg):
+    if coverage.version_info <= (5, 0):
+        raise argparse.ArgumentTypeError('Contexts are only supported with coverage.py >= 5.x')
+    if arg != "test":
+        raise argparse.ArgumentTypeError('--cov-context=test is the only supported value')
+    return arg
+
+
 class StoreReport(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         report_type, file = values
@@ -88,6 +97,9 @@ def pytest_addoption(parser):
                          'Default: False')
     group.addoption('--cov-branch', action='store_true', default=None,
                     help='Enable branch coverage.')
+    group.addoption('--cov-context', action='store', metavar='CONTEXT',
+                    type=validate_context,
+                    help='Dynamic contexts to use. "test" for now.')
 
 
 def _prepare_cov_source(cov_source):
@@ -198,6 +210,9 @@ def pytest_sessionstart(self, session):
         elif not self._started:
             self.start(engine.Central)
 
+        if self.options.cov_context == 'test':
+            session.config.pluginmanager.register(TestContextPlugin(self.cov_controller.cov), '_cov_contexts')
+
     def pytest_configure_node(self, node):
         """Delegate to our implementation.
 
@@ -308,6 +323,24 @@ def pytest_runtest_call(self, item):
             yield
 
 
+class TestContextPlugin(object):
+    def __init__(self, cov):
+        self.cov = cov
+
+    def pytest_runtest_setup(self, item):
+        self.switch_context(item, 'setup')
+
+    def pytest_runtest_teardown(self, item):
+        self.switch_context(item, 'teardown')
+
+    def pytest_runtest_call(self, item):
+        self.switch_context(item, 'run')
+
+    def switch_context(self, item, when):
+        context = "{item.nodeid}|{when}".format(item=item, when=when)
+        self.cov.switch_context(context)
+
+
 @pytest.fixture
 def no_cover():
     """A pytest fixture to disable coverage."""

tests/contextful.py

@@ -0,0 +1,105 @@
+# A test file for test_pytest_cov.py:test_contexts
+
+import unittest
+
+import pytest
+
+
+def test_01():
+    assert 1 == 1                                   # r1
+
+
+def test_02():
+    assert 2 == 2                                   # r2
+
+
+class OldStyleTests(unittest.TestCase):
+    items = []
+
+    @classmethod
+    def setUpClass(cls):
+        cls.items.append("hello")                   # s3
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.items.pop()                             # t4
+
+    def setUp(self):
+        self.number = 1                             # r3 r4
+
+    def tearDown(self):
+        self.number = None                          # r3 r4
+
+    def test_03(self):
+        assert self.number == 1                     # r3
+        assert self.items[0] == "hello"             # r3
+
+    def test_04(self):
+        assert self.number == 1                     # r4
+        assert self.items[0] == "hello"             # r4
+
+
+@pytest.fixture
+def some_data():
+    return [1, 2, 3]                                # s5 s6
+
+
+def test_05(some_data):
+    assert len(some_data) == 3                      # r5
+
+
+@pytest.fixture
+def more_data(some_data):
+    return [2*x for x in some_data]                 # s6
+
+
+def test_06(some_data, more_data):
+    assert len(some_data) == len(more_data)         # r6
+
+
+@pytest.fixture(scope='session')
+def expensive_data():
+    return list(range(10))                          # s7
+
+
+def test_07(expensive_data):
+    assert len(expensive_data) == 10                # r7
+
+
+def test_08(expensive_data):
+    assert len(expensive_data) == 10                # r8
+
+
+@pytest.fixture(params=[1, 2, 3])
+def parametrized_number(request):
+    return request.param                            # s9-1 s9-2 s9-3
+
+
+def test_09(parametrized_number):
+    assert parametrized_number > 0                  # r9-1 r9-2 r9-3
+
+
+def test_10():
+    assert 1 == 1                                   # r10
+
+
+@pytest.mark.parametrize("x, ans", [
+    (1, 101),
+    (2, 202),
+])
+def test_11(x, ans):
+    assert 100 * x + x == ans                       # r11-1 r11-2
+
+
+@pytest.mark.parametrize("x, ans", [
+    (1, 101),
+    (2, 202),
+], ids=['one', 'two'])
+def test_12(x, ans):
+    assert 100 * x + x == ans                       # r12-1 r12-2
+
+
+@pytest.mark.parametrize("x", [1, 2])
+@pytest.mark.parametrize("y", [3, 4])
+def test_13(x, y):
+    assert x + y > 0                                # r13-1 r13-2 r13-3 r13-4