mirror of
https://github.com/official-stockfish/Stockfish.git
synced 2026-07-22 12:47:08 +00:00
Fix test harness timeout enforcement
Fixes #6881. `timeout_decorator()` used a `ThreadPoolExecutor` context manager around blocking output waits. When `future.result(timeout=...)` timed out, leaving the context manager still waited for the worker thread to finish, so a blocked stdout read could keep the instrumented tests hanging past the configured timeout. This change removes that executor wrapper for interactive Stockfish output waits. The harness now drains process output on a daemon reader thread, queues received lines, and applies the deadline directly while waiting for the next queued line. `TimeoutException` also initializes the base exception message so failures show useful text. Validation: - `python3 -m py_compile tests/testing.py tests/instrumented.py` - local timeout smoke test: a 0.2s no-output wait raises in ~0.204s - Stockfish smoke test: startup/`uciok` read succeeds, deliberate no-output wait raises in ~0.205s, engine exits 0 - `make -C src -j4 build` - `../tests/signature.sh` -> `2814421` closes https://github.com/official-stockfish/Stockfish/pull/6882 No functional change
This commit is contained in:
committed by
Joost VandeVondele
parent
1eff8b0389
commit
415ff793a0
@@ -16,6 +16,7 @@ Adrian Petrescu (apetresc)
|
|||||||
Adrian (AdrianGHUB15)
|
Adrian (AdrianGHUB15)
|
||||||
Ahmed Kerimov (wcdbmv)
|
Ahmed Kerimov (wcdbmv)
|
||||||
Ajith Chandy Jose (ajithcj)
|
Ajith Chandy Jose (ajithcj)
|
||||||
|
AK-Khan02
|
||||||
Alain Savard (Rocky640)
|
Alain Savard (Rocky640)
|
||||||
Alayan Feh (Alayan-stk-2)
|
Alayan Feh (Alayan-stk-2)
|
||||||
Alexander Kure
|
Alexander Kure
|
||||||
|
|||||||
+43
-32
@@ -6,12 +6,12 @@ import time
|
|||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
import fnmatch
|
import fnmatch
|
||||||
from functools import wraps
|
|
||||||
from contextlib import redirect_stdout
|
from contextlib import redirect_stdout
|
||||||
import io
|
import io
|
||||||
import tarfile
|
import tarfile
|
||||||
import pathlib
|
import pathlib
|
||||||
import concurrent.futures
|
import queue
|
||||||
|
import threading
|
||||||
import tempfile
|
import tempfile
|
||||||
import shutil
|
import shutil
|
||||||
import requests
|
import requests
|
||||||
@@ -123,7 +123,8 @@ class OrderedClassMembers(type):
|
|||||||
|
|
||||||
|
|
||||||
class TimeoutException(Exception):
|
class TimeoutException(Exception):
|
||||||
def __init__(self, message: str, timeout: int):
|
def __init__(self, message: str, timeout: float):
|
||||||
|
super().__init__(message)
|
||||||
self.message = message
|
self.message = message
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
|
|
||||||
@@ -133,26 +134,6 @@ class UnexpectedOutputException(Exception):
|
|||||||
self.expected = expected
|
self.expected = expected
|
||||||
|
|
||||||
|
|
||||||
def timeout_decorator(timeout: float):
|
|
||||||
def decorator(func):
|
|
||||||
@wraps(func)
|
|
||||||
def wrapper(*args, **kwargs):
|
|
||||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
||||||
future = executor.submit(func, *args, **kwargs)
|
|
||||||
try:
|
|
||||||
result = future.result(timeout=timeout)
|
|
||||||
except concurrent.futures.TimeoutError:
|
|
||||||
raise TimeoutException(
|
|
||||||
f"Function {func.__name__} timed out after {timeout} seconds",
|
|
||||||
timeout,
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
class MiniTestFramework:
|
class MiniTestFramework:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.passed_test_suites = 0
|
self.passed_test_suites = 0
|
||||||
@@ -305,6 +286,8 @@ class Stockfish:
|
|||||||
self.cli = cli
|
self.cli = cli
|
||||||
self.prefix = prefix
|
self.prefix = prefix
|
||||||
self.output = []
|
self.output = []
|
||||||
|
self.output_queue = queue.Queue()
|
||||||
|
self.reader_thread = None
|
||||||
|
|
||||||
self.start()
|
self.start()
|
||||||
|
|
||||||
@@ -336,6 +319,21 @@ class Stockfish:
|
|||||||
universal_newlines=True,
|
universal_newlines=True,
|
||||||
bufsize=1,
|
bufsize=1,
|
||||||
)
|
)
|
||||||
|
self.reader_thread = threading.Thread(
|
||||||
|
target=self._read_process_output, daemon=True
|
||||||
|
)
|
||||||
|
self.reader_thread.start()
|
||||||
|
|
||||||
|
def _read_process_output(self):
|
||||||
|
try:
|
||||||
|
for line in self.process.stdout:
|
||||||
|
line = line.strip()
|
||||||
|
self.output.append(line)
|
||||||
|
self.output_queue.put(line)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
self.output_queue.put(None)
|
||||||
|
|
||||||
def setoption(self, name: str, value: str):
|
def setoption(self, name: str, value: str):
|
||||||
self.send_command(f"setoption name {name} value {value}")
|
self.send_command(f"setoption name {name} value {value}")
|
||||||
@@ -349,31 +347,26 @@ class Stockfish:
|
|||||||
self.process.stdin.write(command + "\n")
|
self.process.stdin.write(command + "\n")
|
||||||
self.process.stdin.flush()
|
self.process.stdin.flush()
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def equals(self, expected_output: str):
|
def equals(self, expected_output: str):
|
||||||
for line in self.readline():
|
for line in self.readline():
|
||||||
if line == expected_output:
|
if line == expected_output:
|
||||||
return
|
return
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def expect(self, expected_output: str):
|
def expect(self, expected_output: str):
|
||||||
for line in self.readline():
|
for line in self.readline():
|
||||||
if fnmatch.fnmatch(line, expected_output):
|
if fnmatch.fnmatch(line, expected_output):
|
||||||
return
|
return
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def contains(self, expected_output: str):
|
def contains(self, expected_output: str):
|
||||||
for line in self.readline():
|
for line in self.readline():
|
||||||
if expected_output in line:
|
if expected_output in line:
|
||||||
return
|
return
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def starts_with(self, expected_output: str):
|
def starts_with(self, expected_output: str):
|
||||||
for line in self.readline():
|
for line in self.readline():
|
||||||
if line.startswith(expected_output):
|
if line.startswith(expected_output):
|
||||||
return
|
return
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def check_output(self, callback):
|
def check_output(self, callback):
|
||||||
if not callback:
|
if not callback:
|
||||||
raise ValueError("Callback function is required")
|
raise ValueError("Callback function is required")
|
||||||
@@ -382,7 +375,6 @@ class Stockfish:
|
|||||||
if callback(line) == True:
|
if callback(line) == True:
|
||||||
return
|
return
|
||||||
|
|
||||||
@timeout_decorator(MAX_TIMEOUT)
|
|
||||||
def expect_for_line_matching(self, line_match: str, expected: str):
|
def expect_for_line_matching(self, line_match: str, expected: str):
|
||||||
for line in self.readline():
|
for line in self.readline():
|
||||||
if fnmatch.fnmatch(line, line_match):
|
if fnmatch.fnmatch(line, line_match):
|
||||||
@@ -391,14 +383,33 @@ class Stockfish:
|
|||||||
else:
|
else:
|
||||||
raise UnexpectedOutputException(line, expected)
|
raise UnexpectedOutputException(line, expected)
|
||||||
|
|
||||||
def readline(self):
|
def readline(self, timeout: float = MAX_TIMEOUT):
|
||||||
if not self.process:
|
if not self.process:
|
||||||
raise RuntimeError("Stockfish process is not started")
|
raise RuntimeError("Stockfish process is not started")
|
||||||
|
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
self._check_process_alive()
|
self._check_process_alive()
|
||||||
line = self.process.stdout.readline().strip()
|
|
||||||
self.output.append(line)
|
remaining_time = deadline - time.monotonic()
|
||||||
|
if remaining_time <= 0:
|
||||||
|
raise TimeoutException(
|
||||||
|
f"No matching output received after {timeout} seconds",
|
||||||
|
timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
line = self.output_queue.get(timeout=remaining_time)
|
||||||
|
except queue.Empty:
|
||||||
|
raise TimeoutException(
|
||||||
|
f"No matching output received after {timeout} seconds",
|
||||||
|
timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
if line is None:
|
||||||
|
self._check_process_alive()
|
||||||
|
raise RuntimeError("Stockfish process has terminated")
|
||||||
|
|
||||||
yield line
|
yield line
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user