73 lines
2.2 KiB
Python
73 lines
2.2 KiB
Python
|
import shlex
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import textwrap
|
||
|
|
import uuid
|
||
|
|
from collections.abc import Iterable
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from mypy.test.config import test_data_prefix
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class PytestResult:
|
||
|
|
input: str
|
||
|
|
input_updated: str # any updates made by --update-data
|
||
|
|
stdout: str
|
||
|
|
stderr: str
|
||
|
|
|
||
|
|
|
||
|
|
def dedent_docstring(s: str) -> str:
|
||
|
|
return textwrap.dedent(s).lstrip()
|
||
|
|
|
||
|
|
|
||
|
|
def run_pytest_data_suite(
|
||
|
|
data_suite: str,
|
||
|
|
*,
|
||
|
|
data_file_prefix: str = "check",
|
||
|
|
pytest_node_prefix: str = "mypy/test/testcheck.py::TypeCheckSuite",
|
||
|
|
extra_args: Iterable[str],
|
||
|
|
max_attempts: int,
|
||
|
|
) -> PytestResult:
|
||
|
|
"""
|
||
|
|
Runs a suite of data test cases through pytest until either tests pass
|
||
|
|
or until a maximum number of attempts (needed for incremental tests).
|
||
|
|
|
||
|
|
:param data_suite: the actual "suite" i.e. the contents of a .test file
|
||
|
|
"""
|
||
|
|
p_test_data = Path(test_data_prefix)
|
||
|
|
p_root = p_test_data.parent.parent
|
||
|
|
p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test"
|
||
|
|
assert not p.exists()
|
||
|
|
data_suite = dedent_docstring(data_suite)
|
||
|
|
try:
|
||
|
|
p.write_text(data_suite)
|
||
|
|
|
||
|
|
test_nodeid = f"{pytest_node_prefix}::{p.name}"
|
||
|
|
extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid]
|
||
|
|
cmd = shlex.join(extra_args)
|
||
|
|
for i in range(max_attempts - 1, -1, -1):
|
||
|
|
print(f">> {cmd}")
|
||
|
|
proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root)
|
||
|
|
if proc.returncode == 0:
|
||
|
|
break
|
||
|
|
prefix = "NESTED PYTEST STDOUT"
|
||
|
|
for line in proc.stdout.decode().splitlines():
|
||
|
|
print(f"{prefix}: {line}")
|
||
|
|
prefix = " " * len(prefix)
|
||
|
|
prefix = "NESTED PYTEST STDERR"
|
||
|
|
for line in proc.stderr.decode().splitlines():
|
||
|
|
print(f"{prefix}: {line}")
|
||
|
|
prefix = " " * len(prefix)
|
||
|
|
print(f"Exit code {proc.returncode} ({i} attempts remaining)")
|
||
|
|
|
||
|
|
return PytestResult(
|
||
|
|
input=data_suite,
|
||
|
|
input_updated=p.read_text(),
|
||
|
|
stdout=proc.stdout.decode(),
|
||
|
|
stderr=proc.stderr.decode(),
|
||
|
|
)
|
||
|
|
finally:
|
||
|
|
p.unlink()
|