Python pytest.Item() Examples

def pytest_runtest_teardown(item: pytest.Item):
    """Hook to run after every test."""
    # Inject footer at end of test, may be followed by additional teardown.
    # Don't do this when running in teamcity, where it's redundant.
    if not teamcity.is_running_under_teamcity():
        global start_time
        duration = time.time() - start_time
        start_time = 0
        print(
            """
==========
======= END: {}::{} ({})
==========""".format(
                sdk_diag.get_test_suite_name(item), item.name, sdk_utils.pretty_duration(duration)
            )
        ) 

def pytest_runtest_setup(item: pytest.Item):
    """Hook to run before every test."""
    # Inject header at start of test, following automatic "path/to/test_file.py::test_name":
    # Don't do this when running in teamcity, where it's redundant.
    if not teamcity.is_running_under_teamcity():
        global start_time
        start_time = time.time()
        print(
            """
==========
======= START: {}::{}
==========""".format(
                sdk_diag.get_test_suite_name(item), item.name
            )
        )

    if INTEGRATION_TEST_LOG_COLLECTION:
        sdk_diag.handle_test_setup(item)
    sdk_utils.check_dcos_min_version_mark(item) 

def _whitelisted_service_names(item: pytest.Item) -> Set[str]:
    """Returns a set of whitelisted service names configured by pytest marker diag_service_whitelist,
    which should be used like this:

    @pytest.mark.diag_service_whitelist(set('service1', 'service2'))
    def your_test_here(): ...

    Note that the diag_service_whitelist marker can be used on function, class, or module
    to be able to hierarchically configure the whitelist.
    """
    if item.get_closest_marker(name='diag_service_whitelist') is None:
        return set()

    whitelisted_service_names: Set[str] = set()
    for mark in item.iter_markers(name='diag_service_whitelist'):
        whitelisted_service_names = whitelisted_service_names.union(mark.args[0])

    return whitelisted_service_names 

def _dump_plans(item: pytest.Item, service_name: str) -> None:
    """If the test had failed, writes the plan state(s) to log file(s)."""

    # Use brief timeouts, we just want a best-effort attempt here:
    plan_names = sdk_plan.list_plans(service_name, 5)
    for plan_name in plan_names:
        plan = sdk_plan.get_plan(service_name, plan_name, 5)
        # Include service name in plan filename, but be careful about folders...
        out_path = _setup_artifact_path(
            item, "plan_{}_{}.json".format(service_name.replace("/", "_"), plan_name)
        )
        out_content = json.dumps(plan, indent=2)
        log.info("=> Writing {} ({} bytes)".format(out_path, len(out_content)))
        with open(out_path, "w") as f:
            f.write(out_content)
            f.write("\n")  # ... and a trailing newline 

def _setup_artifact_path(item: pytest.Item, artifact_name: str) -> str:
    """Given the pytest item and an artifact_name,
    Returns the path to write an artifact with that name."""

    # full item.listchain() is e.g.:
    # - ['build', 'frameworks/template/tests/test_sanity.py', 'test_install']
    # - ['build', 'tests/test_sanity.py', 'test_install']
    # we want to turn both cases into: 'logs/test_sanity_py/test_install'
    if _testlogs_test_index > 0:
        # test_index is defined: get name like "05__test_placement_rules"
        test_name = "{:02d}__{}".format(_testlogs_test_index, item.name)
    else:
        # test_index is not defined: fall back to just "test_placement_rules"
        test_name = item.name

    output_dir = os.path.join(_test_suite_artifact_directory(item), test_name)
    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)

    return os.path.join(output_dir, artifact_name) 

def check_dcos_min_version_mark(item: pytest.Item) -> None:
    """Enforces the dcos_min_version pytest annotation, which should be used like this:

    @pytest.mark.dcos_min_version('1.10')
    def your_test_here(): ...

    In order for this annotation to take effect, this function must be called by a pytest_runtest_setup() hook.
    """
    min_version_mark = item.get_closest_marker("dcos_min_version")
    if min_version_mark:
        min_version = min_version_mark.args[0]
        message = "Feature only supported in DC/OS {} and up".format(min_version)
        if "reason" in min_version_mark.kwargs:
            message += ": {}".format(min_version_mark.kwargs["reason"])
        if dcos_version_less_than(min_version):
            pytest.skip(message) 

def dummy_yaml_custom_test(testdir):
    """Writes a conftest file that collects and executes a dummy yaml test.

    Taken from the docs, but stripped down to the bare minimum, useful for
    tests which needs custom items collected.
    """
    testdir.makeconftest(
        """
        import pytest

        def pytest_collect_file(parent, path):
            if path.ext == ".yaml" and path.basename.startswith("test"):
                return YamlFile(path, parent)

        class YamlFile(pytest.File):
            def collect(self):
                yield YamlItem(self.fspath.basename, self)

        class YamlItem(pytest.Item):
            def runtest(self):
                pass
    """
    )
    testdir.makefile(".yaml", test1="") 

def test_multiple_items_per_collector_byid(self, testdir):
        c = testdir.makeconftest(
            """
            import pytest
            class MyItem(pytest.Item):
                def runtest(self):
                    pass
            class MyCollector(pytest.File):
                def collect(self):
                    return [MyItem(name="xyz", parent=self)]
            def pytest_collect_file(path, parent):
                if path.basename.startswith("conftest"):
                    return MyCollector(path, parent)
        """
        )
        result = testdir.runpytest(c.basename + "::" + "xyz")
        assert result.ret == 0
        result.stdout.fnmatch_lines(["*1 pass*"]) 

def test_xfail_item(testdir):
    # Ensure pytest.xfail works with non-Python Item
    testdir.makeconftest(
        """
        import pytest

        class MyItem(pytest.Item):
            nodeid = 'foo'
            def runtest(self):
                pytest.xfail("Expected Failure")

        def pytest_collect_file(path, parent):
            return MyItem("foo", parent)
    """
    )
    result = testdir.inline_run()
    passed, skipped, failed = result.listoutcomes()
    assert not failed
    xfailed = [r for r in skipped if hasattr(r, "wasxfail")]
    assert xfailed 

def test_mark_xfail_item(testdir):
    # Ensure pytest.mark.xfail works with non-Python Item
    testdir.makeconftest(
        """
        import pytest

        class MyItem(pytest.Item):
            nodeid = 'foo'
            def setup(self):
                marker = pytest.mark.xfail("1 == 2", reason="Expected failure - false")
                self.add_marker(marker)
                marker = pytest.mark.xfail(True, reason="Expected failure - true")
                self.add_marker(marker)
            def runtest(self):
                assert False

        def pytest_collect_file(path, parent):
            return MyItem("foo", parent)
    """
    )
    result = testdir.inline_run()
    passed, skipped, failed = result.listoutcomes()
    assert not failed
    xfailed = [r for r in skipped if hasattr(r, "wasxfail")]
    assert xfailed 

def __init__(self, py_test_service,
                 log_level=logging.NOTSET,
                 endpoint=None):
        """Initialize RPReport Listener instance.

        :param py_test_service: PyTestServiceClass instance
        :param log_level:       One of the 'CRITICAL', 'ERROR',
                                'WARNING','INFO','DEBUG', 'NOTSET'
        :param endpoint:        Report Portal API endpoint
        """
        # Test Item result
        self.PyTestService = py_test_service
        self.result = None
        self.issue = {}
        self._log_level = log_level
        if PYTEST_HAS_LOGGING_PLUGIN:
            self._log_handler = \
                RPLogHandler(py_test_service=py_test_service,
                             level=log_level,
                             filter_client_logs=True,
                             endpoint=endpoint) 

def pytest_runtest_protocol(self, item):
        """
        Adding issues id marks to the test item.

        :param item:  Pytest.Item
        :return: generator object
        """
        self._add_issue_id_marks(item)
        item_id = self.PyTestService.start_pytest_item(item)
        if PYTEST_HAS_LOGGING_PLUGIN:
            # This check can go away once we support pytest >= 3.3
            with patching_logger_class():
                with _pytest.logging.catching_logs(self._log_handler,
                                                   level=self._log_level):
                    yield
        else:
            yield
        # Finishing item in RP
        self.PyTestService.finish_pytest_item(
            item, item_id, self.result or 'SKIPPED', self.issue or None) 

def _get_item_parts(item):
        """
        Get item of parents.

        :param item: pytest.Item
        :return list of parents
        """
        parts = []
        parent = item.parent
        if not isinstance(parent, Instance):
            parts.append(parent)
        while True:
            parent = parent.parent
            if parent is None:
                break
            if isinstance(parent, Instance):
                continue
            if isinstance(parent, Session):
                break
            parts.append(parent)

        parts.reverse()
        return parts 

def _get_item_dirs(item):
        """
        Get directory of item.

        :param item: pytest.Item
        :return: list of dirs
        """
        root_path = item.session.config.rootdir.strpath
        dir_path = item.fspath.new(basename="")
        rel_dir = dir_path.new(dirname=dir_path.relto(root_path), basename="",
                               drive="")

        dir_list = []
        for directory in rel_dir.parts(reverse=False):
            dir_name = directory.basename
            if dir_name:
                dir_list.append(dir_name)

        return dir_list 

def _get_item_name(test_item):
        """
        Get name of item.

        :param test_item: pytest.Item
        :return: name
        """
        name = test_item._rp_name
        if len(name) > 256:
            name = name[:256]
            test_item.warn(
                PytestWarning(
                    'Test node ID was truncated to "{}" because of name size '
                    'constrains on reportportal'.format(name)
                )
            )
        return name 

def pytest_collection_modifyitems(self, session, config, items):
        if self.active:
            new_items = OrderedDict()
            other_items = OrderedDict()
            for item in items:
                if item.nodeid not in self.cached_nodeids:
                    new_items[item.nodeid] = item
                else:
                    other_items[item.nodeid] = item

            items[:] = self._get_increasing_order(
                new_items.values()
            ) + self._get_increasing_order(other_items.values())
        self.cached_nodeids = [x.nodeid for x in items if isinstance(x, pytest.Item)] 

def pytest_collectreport(self, report):
        if report.failed:
            self.stats.setdefault("error", []).append(report)
        elif report.skipped:
            self.stats.setdefault("skipped", []).append(report)
        items = [x for x in report.result if isinstance(x, pytest.Item)]
        self._numcollected += len(items)
        if self.isatty:
            self.report_collect() 

def pytest_collection_modifyitems(self, session, config, items):
        if self.active:
            new_items = OrderedDict()
            other_items = OrderedDict()
            for item in items:
                if item.nodeid not in self.cached_nodeids:
                    new_items[item.nodeid] = item
                else:
                    other_items[item.nodeid] = item

            items[:] = self._get_increasing_order(
                new_items.values()
            ) + self._get_increasing_order(other_items.values())
        self.cached_nodeids = [x.nodeid for x in items if isinstance(x, pytest.Item)] 

def pytest_collectreport(self, report):
        if report.failed:
            self.stats.setdefault("error", []).append(report)
        elif report.skipped:
            self.stats.setdefault("skipped", []).append(report)
        items = [x for x in report.result if isinstance(x, pytest.Item)]
        self._numcollected += len(items)
        if self.isatty:
            self.report_collect() 

def pytest_runtest_setup(item: pytest.Item):

    if tuple(item.iter_markers(name="indy")) and not INDY_FOUND:
        pytest.skip("test requires Indy support")

    if tuple(item.iter_markers(name="postgres")) and not POSTGRES_URL:
        pytest.skip("test requires Postgres support") 

def pytest_runtest_makereport(item: pytest.Item, call):  # _pytest.runner.CallInfo
    """Hook to run after every test, before any other post-test hooks.
    See also: https://docs.pytest.org/en/latest/example/simple.html\
    #making-test-result-information-available-in-fixtures
    """

    # Execute all other hooks to obtain the report object.
    outcome = yield

    # Handle failures. Must be done here and not in a fixture in order to properly handle post-yield
    # fixture teardown failures.
    if INTEGRATION_TEST_LOG_COLLECTION:

        @retrying.retry(
            # It is possible that test-related timeouts are triggered while diagnostics-fetching
            # runs, instead of when the actual test runs. That manifests as an "INTERNALERROR>
            # Failed: Timeout" that crashes the test process. That is undesirable because then we
            # get no diagnostics artifacts which would allow us to investigate why the test timed
            # out in the first place.
            #
            # Here, we retry the diagnostics-fetching method in case of pytest exceptions.
            #
            # Check DCOS-51362 for more context.
            stop_max_attempt_number=3,
            retry_on_exception=lambda e: isinstance(e, pytest.fail.Exception),
        )
        def _create_diagnostics_bundle():
            sdk_diag.handle_test_report(item, outcome.get_result())

        _create_diagnostics_bundle()

    else:
        print("INTEGRATION_TEST_LOG_COLLECTION==False. Skipping log collection") 

def get_test_suite_name(item: pytest.Item) -> str:
    """Returns the test suite name to use for a given test."""
    # frameworks/template/tests/test_sanity.py => test_sanity_py
    # tests/test_sanity.py => test_sanity_py

    # use the class name as the suite name if item is a method
    if inspect.ismethod(item.obj):
        return str(os.path.basename(item.getparent(pytest.Class).name)).replace(".", "_")

    return str(os.path.basename(item.parent.name)).replace(".", "_") 

def handle_test_setup(item: pytest.Item) -> None:
    """Does some initialization at the start of a test.

    This should be called in a pytest_runtest_setup() hook.
    See also handle_failed_test() which must be called from a pytest_runtest_makereport() hookimpl hook."""

    # Check if we're entering a new test suite.
    global _testlogs_test_index
    global _testlogs_current_test_suite
    test_suite = get_test_suite_name(item)
    if test_suite != _testlogs_current_test_suite:
        # New test suite:
        # 1 Store all the task ids which already exist as of this point.
        _testlogs_current_test_suite = test_suite
        global _testlogs_ignored_task_ids
        _testlogs_ignored_task_ids = _testlogs_ignored_task_ids.union(
            [
                task.id
                for task in sdk_tasks.get_summary(with_completed=True)
                if not _task_whitelist_callback(task)
            ]
        )
        log.info(
            "Entering new test suite {}: {} preexisting tasks will be ignored on test failure.".format(
                test_suite, len(_testlogs_ignored_task_ids)
            )
        )
        # 2 Reset the test index.
        _testlogs_test_index = 0
        # 3 Remove any prior logs for the test suite.
        test_log_dir = _test_suite_artifact_directory(item)
        if os.path.exists(test_log_dir):
            log.info("Deleting existing test suite logs: {}/".format(test_log_dir))
            shutil.rmtree(test_log_dir)

    # Increment the test index (to 1, if this is a new suite)
    _testlogs_test_index += 1 

def _dump_threads(item: pytest.Item, service_name: str) -> None:
    threads = sdk_cmd.service_request(
        "GET", service_name, "v1/debug/threads", timeout_seconds=5
    ).text
    out_path = _setup_artifact_path(item, "threads_{}.txt".format(service_name.replace("/", "_")))
    log.info("=> Writing {} ({} bytes)".format(out_path, len(threads)))
    with open(out_path, "w") as f:
        f.write(threads)
        f.write("\n")  # ... and a trailing newline 

def _dump_diagnostics_bundle(item: pytest.Item) -> None:
    """Creates and downloads a DC/OS diagnostics bundle, and saves it to the artifact path for this test."""
    rc, _, _ = sdk_cmd.run_cli("node diagnostics create all")
    if rc:
        log.error("Diagnostics bundle creation failed.")
        return

    @retrying.retry(
        wait_fixed=5000,
        stop_max_delay=10 * 60 * 1000,
        retry_on_result=lambda result: result is None,
    )
    def wait_for_bundle_file() -> Optional[str]:
        rc, stdout, stderr = sdk_cmd.run_cli("node diagnostics --status --json")
        if rc:
            return None

        # e.g. { "some-ip": { stuff we want } }
        status = next(iter(json.loads(stdout).values()))
        if status["job_progress_percentage"] != 100:
            return None

        # e.g. "/var/lib/dcos/dcos-diagnostics/diag-bundles/bundle-2018-01-11-1515698691.zip"
        return str(os.path.basename(status["last_bundle_dir"]))

    bundle_filename = str(wait_for_bundle_file())
    if bundle_filename:
        sdk_cmd.run_cli(
            "node diagnostics download {} --location={}".format(
                bundle_filename, _setup_artifact_path(item, bundle_filename)
            )
        )
    else:
        log.error("Diagnostics bundle didnt finish in time, giving up.") 

def _dump_mesos_state(item: pytest.Item) -> None:
    """Downloads state from the Mesos master and saves it to the artifact path for this test."""
    for name in ["state.json", "slaves"]:
        r = sdk_cmd.cluster_request("GET", "/mesos/{}".format(name), raise_on_error=False)
        if r.ok:
            if name.endswith(".json"):
                name = name[: -len(".json")]  # avoid duplicate '.json'
            with open(_setup_artifact_path(item, "mesos_{}.json".format(name)), "w") as f:
                f.write(r.text) 

def _dump_task_logs_for_agent(item: pytest.Item, agent_id: str, agent_tasks: List[_TaskEntry]) -> None:
    agent_executor_paths = sdk_cmd.cluster_request(
        "GET", "/slave/{}/files/debug".format(agent_id)
    ).json()
    task_byte_count = 0
    for task_entry in agent_tasks:
        try:
            task_byte_count += _dump_task_logs_for_task(
                item, agent_id, agent_executor_paths, task_entry
            )
        except Exception:
            log.exception("Failed to get logs for task {}".format(task_entry))
    log.info(
        "Downloaded {} bytes of logs from {} tasks on agent {}".format(
            task_byte_count, len(agent_tasks), agent_id
        )
    )

    # fetch agent log separately due to its totally different fetch semantics vs the task/executor logs
    if "/slave/log" in agent_executor_paths:
        out_path = _setup_artifact_path(item, "agent_{}.log".format(agent_id))
        stream = sdk_cmd.cluster_request(
            "GET", "/slave/{}/files/download?path=/slave/log".format(agent_id), stream=True
        )
        with open(out_path, "wb") as f:
            for chunk in stream.iter_content(chunk_size=8192):
                f.write(chunk) 

def _select_log_files(
    item: pytest.Item,
    task_id: str,
    file_infos: List[Dict[str, Any]],
    source: str,
    selected: 'collections.OrderedDict[str, Any]',
) -> None:
    """Finds and produces the 'stderr'/'stdout' file entries from the provided directory list returned by the agent.

    Results are placed in the 'selected' param.
    """
    logfile_pattern = re.compile(r"^.*/(stdout|stderr)(\.[0-9]+)?$")
    for file_info in file_infos:
        if not logfile_pattern.match(file_info["path"]):
            continue
        # Example output filename (sort by time):
        # 180125_225944.world-1-server__4d534510-35d9-4f06-811e-e9a9ffa4d14f.task.stdout.log
        # 180126_000225.hello-0-server__15174696-2d3d-48e9-b492-d9a0cc289786.executor.stderr.log
        # 180126_002024.hello-world.662e7976-0224-11e8-b2f2-deead5f2b92b.stdout.1.log
        out_filename = "{}.{}.{}{}.log".format(
            time.strftime("%y%m%d_%H%M%S", time.gmtime(file_info["mtime"])),
            task_id,
            source,
            os.path.basename(file_info["path"]),
        )
        selected[_setup_artifact_path(item, out_filename)] = file_info 

def _test_suite_artifact_directory(item: pytest.Item) -> str:
    """Returns the parent directory for the artifacts across a suite of tests."""
    return os.path.join("logs", get_test_suite_name(item)) 

def __init__(self, pyfile, outfile, parent):
        pytest.Item.__init__(self, str(pyfile), parent)
        self.pyfile = pyfile
        self.outfile = outfile