add proxy gateways with --px arg

Proxy gateways do not run workers, and are meant to be passed with the
`via` attribute to additional gateways.
They are useful for running multiple workers on remote machines.

Example usage:
```
pytest -sv --dist=load --px "id=my_proxy//socket=IP:PORT" --tx "5*popen//via=my_proxy"
```

Proxy gateways do not run workers, anda re meant to be passed
This commit is contained in:
Alon Livne 2025-01-11 17:02:13 +02:00
parent a82981f444
commit a9a303f688
7 changed files with 35 additions and 14 deletions

View File

@ -139,6 +139,17 @@ def pytest_addoption(parser: pytest.Parser) -> None:
"--tx ssh=user@codespeak.net//chdir=testcache" "--tx ssh=user@codespeak.net//chdir=testcache"
), ),
) )
group.addoption(
"--px",
dest="px",
action="append",
default=[],
metavar="xspec",
help=(
"Add a proxy gateway to pass to test execution environments using `via`. Example:\n"
"--px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy"
),
)
group._addoption( group._addoption(
"-d", "-d",
action="store_true", action="store_true",

View File

@ -6,7 +6,7 @@ import pytest
from xdist.remote import Producer from xdist.remote import Producer
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config from xdist.workermanage import parse_tx_spec_config
from xdist.workermanage import WorkerController from xdist.workermanage import WorkerController
@ -26,7 +26,7 @@ class EachScheduling:
def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
self.config = config self.config = config
self.numnodes = len(parse_spec_config(config)) self.numnodes = len(parse_tx_spec_config(config))
self.node2collection: dict[WorkerController, list[str]] = {} self.node2collection: dict[WorkerController, list[str]] = {}
self.node2pending: dict[WorkerController, list[int]] = {} self.node2pending: dict[WorkerController, list[int]] = {}
self._started: list[WorkerController] = [] self._started: list[WorkerController] = []

View File

@ -7,7 +7,7 @@ import pytest
from xdist.remote import Producer from xdist.remote import Producer
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config from xdist.workermanage import parse_tx_spec_config
from xdist.workermanage import WorkerController from xdist.workermanage import WorkerController
@ -58,7 +58,7 @@ class LoadScheduling:
""" """
def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
self.numnodes = len(parse_spec_config(config)) self.numnodes = len(parse_tx_spec_config(config))
self.node2collection: dict[WorkerController, list[str]] = {} self.node2collection: dict[WorkerController, list[str]] = {}
self.node2pending: dict[WorkerController, list[int]] = {} self.node2pending: dict[WorkerController, list[int]] = {}
self.pending: list[int] = [] self.pending: list[int] = []

View File

@ -8,7 +8,7 @@ import pytest
from xdist.remote import Producer from xdist.remote import Producer
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config from xdist.workermanage import parse_tx_spec_config
from xdist.workermanage import WorkerController from xdist.workermanage import WorkerController
@ -91,7 +91,7 @@ class LoadScopeScheduling:
""" """
def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
self.numnodes = len(parse_spec_config(config)) self.numnodes = len(parse_tx_spec_config(config))
self.collection: list[str] | None = None self.collection: list[str] | None = None
self.workqueue: OrderedDict[str, dict[str, bool]] = OrderedDict() self.workqueue: OrderedDict[str, dict[str, bool]] = OrderedDict()

View File

@ -7,7 +7,7 @@ import pytest
from xdist.remote import Producer from xdist.remote import Producer
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config from xdist.workermanage import parse_tx_spec_config
from xdist.workermanage import WorkerController from xdist.workermanage import WorkerController
@ -65,7 +65,7 @@ class WorkStealingScheduling:
""" """
def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
self.numnodes = len(parse_spec_config(config)) self.numnodes = len(parse_tx_spec_config(config))
self.node2collection: dict[WorkerController, list[str]] = {} self.node2collection: dict[WorkerController, list[str]] = {}
self.node2pending: dict[WorkerController, list[int]] = {} self.node2pending: dict[WorkerController, list[int]] = {}
self.pending: list[int] = [] self.pending: list[int] = []

View File

@ -23,7 +23,7 @@ from xdist.remote import Producer
from xdist.remote import WorkerInfo from xdist.remote import WorkerInfo
def parse_spec_config(config: pytest.Config) -> list[str]: def parse_tx_spec_config(config: pytest.Config) -> list[str]:
xspeclist = [] xspeclist = []
tx: list[str] = config.getvalue("tx") tx: list[str] = config.getvalue("tx")
for xspec in tx: for xspec in tx:
@ -57,8 +57,15 @@ class NodeManager:
if self.testrunuid is None: if self.testrunuid is None:
self.testrunuid = uuid.uuid4().hex self.testrunuid = uuid.uuid4().hex
self.group = execnet.Group(execmodel="main_thread_only") self.group = execnet.Group(execmodel="main_thread_only")
for proxy_spec in self._getpxspecs():
# Proxy gateways do not run workers, and are meant to be passed with the `via` attribute
# to additional gateways.
# They are useful for running multiple workers on remote machines.
if getattr(proxy_spec, "id", None) is None:
raise pytest.UsageError(f"Proxy gateway {proxy_spec} must include an id")
self.group.makegateway(proxy_spec)
if specs is None: if specs is None:
specs = self._getxspecs() specs = self._gettxspecs()
self.specs: list[execnet.XSpec] = [] self.specs: list[execnet.XSpec] = []
for spec in specs: for spec in specs:
if not isinstance(spec, execnet.XSpec): if not isinstance(spec, execnet.XSpec):
@ -107,8 +114,11 @@ class NodeManager:
def teardown_nodes(self) -> None: def teardown_nodes(self) -> None:
self.group.terminate(self.EXIT_TIMEOUT) self.group.terminate(self.EXIT_TIMEOUT)
def _getxspecs(self) -> list[execnet.XSpec]: def _gettxspecs(self) -> list[execnet.XSpec]:
return [execnet.XSpec(x) for x in parse_spec_config(self.config)] return [execnet.XSpec(x) for x in parse_tx_spec_config(self.config)]
def _getpxspecs(self) -> list[execnet.XSpec]:
return [execnet.XSpec(x) for x in self.config.getoption("px")]
def _getrsyncdirs(self) -> list[Path]: def _getrsyncdirs(self) -> list[Path]:
for spec in self.specs: for spec in self.specs:

View File

@ -295,7 +295,7 @@ class TestDistOptions:
def test_getxspecs(self, pytester: pytest.Pytester) -> None: def test_getxspecs(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz") config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
nodemanager = NodeManager(config) nodemanager = NodeManager(config)
xspecs = nodemanager._getxspecs() xspecs = nodemanager._gettxspecs()
assert len(xspecs) == 2 assert len(xspecs) == 2
print(xspecs) print(xspecs)
assert xspecs[0].popen assert xspecs[0].popen
@ -303,7 +303,7 @@ class TestDistOptions:
def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None: def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--tx=3*popen") config = pytester.parseconfigure("--tx=3*popen")
xspecs = NodeManager(config)._getxspecs() xspecs = NodeManager(config)._gettxspecs()
assert len(xspecs) == 3 assert len(xspecs) == 3
assert xspecs[1].popen assert xspecs[1].popen