Fix test harness timeout enforcement

Fixes #6881.

`timeout_decorator()` used a `ThreadPoolExecutor` context manager around blocking output waits. When `future.result(timeout=...)` timed out, leaving the context manager still waited for the worker thread to finish, so a blocked stdout read could keep the instrumented tests hanging past the configured timeout.

This change removes that executor wrapper for interactive Stockfish output waits. The harness now drains process output on a daemon reader thread, queues received lines, and applies the deadline directly while waiting for the next queued line. `TimeoutException` also initializes the base exception message so failures show useful text.

Validation:
- `python3 -m py_compile tests/testing.py tests/instrumented.py`
- local timeout smoke test: a 0.2s no-output wait raises in ~0.204s
- Stockfish smoke test: startup/`uciok` read succeeds, deliberate no-output wait raises in ~0.205s, engine exits 0
- `make -C src -j4 build`
- `../tests/signature.sh` -> `2814421`

closes https://github.com/official-stockfish/Stockfish/pull/6882

No functional change
This commit is contained in:
Abdul Khan
2026-06-09 19:38:00 +02:00
committed by Joost VandeVondele
parent 1eff8b0389
commit 415ff793a0
2 changed files with 44 additions and 32 deletions
+43 -32
View File
@@ -6,12 +6,12 @@ import time
import sys
import traceback
import fnmatch
from functools import wraps
from contextlib import redirect_stdout
import io
import tarfile
import pathlib
import concurrent.futures
import queue
import threading
import tempfile
import shutil
import requests
@@ -123,7 +123,8 @@ class OrderedClassMembers(type):
class TimeoutException(Exception):
def __init__(self, message: str, timeout: int):
def __init__(self, message: str, timeout: float):
super().__init__(message)
self.message = message
self.timeout = timeout
@@ -133,26 +134,6 @@ class UnexpectedOutputException(Exception):
self.expected = expected
def timeout_decorator(timeout: float):
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
with concurrent.futures.ThreadPoolExecutor() as executor:
future = executor.submit(func, *args, **kwargs)
try:
result = future.result(timeout=timeout)
except concurrent.futures.TimeoutError:
raise TimeoutException(
f"Function {func.__name__} timed out after {timeout} seconds",
timeout,
)
return result
return wrapper
return decorator
class MiniTestFramework:
def __init__(self):
self.passed_test_suites = 0
@@ -305,6 +286,8 @@ class Stockfish:
self.cli = cli
self.prefix = prefix
self.output = []
self.output_queue = queue.Queue()
self.reader_thread = None
self.start()
@@ -336,6 +319,21 @@ class Stockfish:
universal_newlines=True,
bufsize=1,
)
self.reader_thread = threading.Thread(
target=self._read_process_output, daemon=True
)
self.reader_thread.start()
def _read_process_output(self):
try:
for line in self.process.stdout:
line = line.strip()
self.output.append(line)
self.output_queue.put(line)
except (OSError, ValueError):
pass
finally:
self.output_queue.put(None)
def setoption(self, name: str, value: str):
self.send_command(f"setoption name {name} value {value}")
@@ -349,31 +347,26 @@ class Stockfish:
self.process.stdin.write(command + "\n")
self.process.stdin.flush()
@timeout_decorator(MAX_TIMEOUT)
def equals(self, expected_output: str):
for line in self.readline():
if line == expected_output:
return
@timeout_decorator(MAX_TIMEOUT)
def expect(self, expected_output: str):
for line in self.readline():
if fnmatch.fnmatch(line, expected_output):
return
@timeout_decorator(MAX_TIMEOUT)
def contains(self, expected_output: str):
for line in self.readline():
if expected_output in line:
return
@timeout_decorator(MAX_TIMEOUT)
def starts_with(self, expected_output: str):
for line in self.readline():
if line.startswith(expected_output):
return
@timeout_decorator(MAX_TIMEOUT)
def check_output(self, callback):
if not callback:
raise ValueError("Callback function is required")
@@ -382,7 +375,6 @@ class Stockfish:
if callback(line) == True:
return
@timeout_decorator(MAX_TIMEOUT)
def expect_for_line_matching(self, line_match: str, expected: str):
for line in self.readline():
if fnmatch.fnmatch(line, line_match):
@@ -391,14 +383,33 @@ class Stockfish:
else:
raise UnexpectedOutputException(line, expected)
def readline(self):
def readline(self, timeout: float = MAX_TIMEOUT):
if not self.process:
raise RuntimeError("Stockfish process is not started")
deadline = time.monotonic() + timeout
while True:
self._check_process_alive()
line = self.process.stdout.readline().strip()
self.output.append(line)
remaining_time = deadline - time.monotonic()
if remaining_time <= 0:
raise TimeoutException(
f"No matching output received after {timeout} seconds",
timeout,
)
try:
line = self.output_queue.get(timeout=remaining_time)
except queue.Empty:
raise TimeoutException(
f"No matching output received after {timeout} seconds",
timeout,
)
if line is None:
self._check_process_alive()
raise RuntimeError("Stockfish process has terminated")
yield line