Fix test harness timeout enforcement

Fixes #6881.

`timeout_decorator()` used a `ThreadPoolExecutor` context manager around blocking output waits. When `future.result(timeout=...)` timed out, leaving the context manager still waited for the worker thread to finish, so a blocked stdout read could keep the instrumented tests hanging past the configured timeout.

This change removes that executor wrapper for interactive Stockfish output waits. The harness now drains process output on a daemon reader thread, queues received lines, and applies the deadline directly while waiting for the next queued line. `TimeoutException` also initializes the base exception message so failures show useful text.

Validation:
- `python3 -m py_compile tests/testing.py tests/instrumented.py`
- local timeout smoke test: a 0.2s no-output wait raises in ~0.204s
- Stockfish smoke test: startup/`uciok` read succeeds, deliberate no-output wait raises in ~0.205s, engine exits 0
- `make -C src -j4 build`
- `../tests/signature.sh` -> `2814421`

closes https://github.com/official-stockfish/Stockfish/pull/6882

No functional change
This commit is contained in:
Abdul Khan
2026-06-09 19:38:00 +02:00
committed by Joost VandeVondele
parent 1eff8b0389
commit 415ff793a0
2 changed files with 44 additions and 32 deletions
+1
View File
@@ -16,6 +16,7 @@ Adrian Petrescu (apetresc)
Adrian (AdrianGHUB15) Adrian (AdrianGHUB15)
Ahmed Kerimov (wcdbmv) Ahmed Kerimov (wcdbmv)
Ajith Chandy Jose (ajithcj) Ajith Chandy Jose (ajithcj)
AK-Khan02
Alain Savard (Rocky640) Alain Savard (Rocky640)
Alayan Feh (Alayan-stk-2) Alayan Feh (Alayan-stk-2)
Alexander Kure Alexander Kure
+43 -32
View File
@@ -6,12 +6,12 @@ import time
import sys import sys
import traceback import traceback
import fnmatch import fnmatch
from functools import wraps
from contextlib import redirect_stdout from contextlib import redirect_stdout
import io import io
import tarfile import tarfile
import pathlib import pathlib
import concurrent.futures import queue
import threading
import tempfile import tempfile
import shutil import shutil
import requests import requests
@@ -123,7 +123,8 @@ class OrderedClassMembers(type):
class TimeoutException(Exception): class TimeoutException(Exception):
def __init__(self, message: str, timeout: int): def __init__(self, message: str, timeout: float):
super().__init__(message)
self.message = message self.message = message
self.timeout = timeout self.timeout = timeout
@@ -133,26 +134,6 @@ class UnexpectedOutputException(Exception):
self.expected = expected self.expected = expected
def timeout_decorator(timeout: float):
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(func, *args, **kwargs)
try:
result = future.result(timeout=timeout)
except concurrent.futures.TimeoutError:
raise TimeoutException(
f"Function {func.__name__} timed out after {timeout} seconds",
timeout,
)
return result
return wrapper
return decorator
class MiniTestFramework: class MiniTestFramework:
def __init__(self): def __init__(self):
self.passed_test_suites = 0 self.passed_test_suites = 0
@@ -305,6 +286,8 @@ class Stockfish:
self.cli = cli self.cli = cli
self.prefix = prefix self.prefix = prefix
self.output = [] self.output = []
self.output_queue = queue.Queue()
self.reader_thread = None
self.start() self.start()
@@ -336,6 +319,21 @@ class Stockfish:
universal_newlines=True, universal_newlines=True,
bufsize=1, bufsize=1,
) )
self.reader_thread = threading.Thread(
target=self._read_process_output, daemon=True
)
self.reader_thread.start()
def _read_process_output(self):
try:
for line in self.process.stdout:
line = line.strip()
self.output.append(line)
self.output_queue.put(line)
except (OSError, ValueError):
pass
finally:
self.output_queue.put(None)
def setoption(self, name: str, value: str): def setoption(self, name: str, value: str):
self.send_command(f"setoption name {name} value {value}") self.send_command(f"setoption name {name} value {value}")
@@ -349,31 +347,26 @@ class Stockfish:
self.process.stdin.write(command + "\n") self.process.stdin.write(command + "\n")
self.process.stdin.flush() self.process.stdin.flush()
@timeout_decorator(MAX_TIMEOUT)
def equals(self, expected_output: str): def equals(self, expected_output: str):
for line in self.readline(): for line in self.readline():
if line == expected_output: if line == expected_output:
return return
@timeout_decorator(MAX_TIMEOUT)
def expect(self, expected_output: str): def expect(self, expected_output: str):
for line in self.readline(): for line in self.readline():
if fnmatch.fnmatch(line, expected_output): if fnmatch.fnmatch(line, expected_output):
return return
@timeout_decorator(MAX_TIMEOUT)
def contains(self, expected_output: str): def contains(self, expected_output: str):
for line in self.readline(): for line in self.readline():
if expected_output in line: if expected_output in line:
return return
@timeout_decorator(MAX_TIMEOUT)
def starts_with(self, expected_output: str): def starts_with(self, expected_output: str):
for line in self.readline(): for line in self.readline():
if line.startswith(expected_output): if line.startswith(expected_output):
return return
@timeout_decorator(MAX_TIMEOUT)
def check_output(self, callback): def check_output(self, callback):
if not callback: if not callback:
raise ValueError("Callback function is required") raise ValueError("Callback function is required")
@@ -382,7 +375,6 @@ class Stockfish:
if callback(line) == True: if callback(line) == True:
return return
@timeout_decorator(MAX_TIMEOUT)
def expect_for_line_matching(self, line_match: str, expected: str): def expect_for_line_matching(self, line_match: str, expected: str):
for line in self.readline(): for line in self.readline():
if fnmatch.fnmatch(line, line_match): if fnmatch.fnmatch(line, line_match):
@@ -391,14 +383,33 @@ class Stockfish:
else: else:
raise UnexpectedOutputException(line, expected) raise UnexpectedOutputException(line, expected)
def readline(self): def readline(self, timeout: float = MAX_TIMEOUT):
if not self.process: if not self.process:
raise RuntimeError("Stockfish process is not started") raise RuntimeError("Stockfish process is not started")
deadline = time.monotonic() + timeout
while True: while True:
self._check_process_alive() self._check_process_alive()
line = self.process.stdout.readline().strip()
self.output.append(line) remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise TimeoutException(
f"No matching output received after {timeout} seconds",
timeout,
)
try:
line = self.output_queue.get(timeout=remaining_time)
except queue.Empty:
raise TimeoutException(
f"No matching output received after {timeout} seconds",
timeout,
)
if line is None:
self._check_process_alive()
raise RuntimeError("Stockfish process has terminated")
yield line yield line