From 9f8733441eef394fb4529d81189af36b92f7e6ae Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 15 Oct 2026 13:35:50 +0300 Subject: [PATCH 1/3] fix: PostgresNode::slow_start rollback code is corrected An easy rollback rule is used: stop -> refresh _manually_started_pm_pid --- src/impl/error_utils.py | 113 ++++++++++++++++++++++++++++++++++ src/node.py | 25 +++++--- tests/test_testgres_common.py | 7 +-- 3 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 src/impl/error_utils.py diff --git a/src/impl/error_utils.py b/src/impl/error_utils.py new file mode 100644 index 00000000..f099afb6 --- /dev/null +++ b/src/impl/error_utils.py @@ -0,0 +1,113 @@ +import traceback +import typing +import logging + + +class ErrorUtils: + @staticmethod + def ExceptionToHumanText(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if __class__.Helper__GetPrevExc(exc) is None: + return __class__.Helper__ExceptionToHumanText__Single(exc) + + return __class__.Helper__ExceptionToHumanText__Chain(exc) + + # -------------------------------------------------------------------- + @staticmethod + def Helper__ExceptionToHumanText__Single(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if isinstance(exc, AssertionError): + err_msg = "Exception ({}).".format(type(exc).__name__) + assert type(err_msg) is str + + exc_info_lines = traceback.format_exception(exc) + assert type(exc_info_lines) is list + err_msg2 = "".join(exc_info_lines).strip() + + if err_msg2 != "": + err_msg += " " + err_msg2 + + assert type(err_msg) is str + return err_msg + + err_msg = "Exception ({})".format(type(exc).__name__) + assert type(err_msg) is str + + err_msg2 = str(exc).strip() + + if err_msg2 == "": + err_msg += "." + else: + err_msg += ": " + err_msg2 + + assert type(err_msg) is str + return err_msg + + # -------------------------------------------------------------------- + @staticmethod + def Helper__ExceptionToHumanText__Chain(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + chain: typing.List[BaseException] = [] + curr = exc + + processed_exc_ids: typing.Set[int] = set() + + while curr is not None: + assert isinstance(curr, BaseException) + if id(curr) in processed_exc_ids: + logging.error("Cycle in exception chain: {}".format( + " --> ".join( + [type(x).__name__ for x in reversed(chain)], + ) + )) + break + + processed_exc_ids.add(id(curr)) + + chain.append(curr) + curr = __class__.Helper__GetPrevExc(curr) + continue + + lines: typing.List[str] = [] + + lines.append("It is a chain of exceptions (len: {}):".format( + len(chain), + )) + + n = 0 + for e in reversed(chain): + n += 1 + line1 = "---- {}. Exception ({})".format(n, type(e).__name__) + line2 = __class__.Helper__GetExcMsg(e) + + if line2 == "": + lines.append(line1) + else: + lines.append(line1 + ":") + lines.append(line2) + continue + + lines.append("--------") + + return "\n".join(lines) + + # -------------------------------------------------------------------- + @staticmethod + def Helper__GetPrevExc(exc: BaseException) -> typing.Optional[BaseException]: + assert isinstance(exc, BaseException) + return exc.__cause__ or exc.__context__ + + # -------------------------------------------------------------------- + @staticmethod + def Helper__GetExcMsg(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if isinstance(exc, AssertionError): + exc_info_lines = traceback.format_exception(exc) + assert type(exc_info_lines) is list + return "".join(exc_info_lines).strip() + + return str(exc) diff --git a/src/node.py b/src/node.py index 4706a287..c6190060 100644 --- a/src/node.py +++ b/src/node.py @@ -55,6 +55,7 @@ from .port_manager import PortManager from .impl.port_manager__this_host import PortManager__ThisHost from .impl.port_manager__generic2 import PortManager__Generic2 +from .impl.error_utils import ErrorUtils from .impl import internal_utils from .logger import TestgresLogger @@ -1183,15 +1184,23 @@ def slow_start( suppress=suppressed_exceptions, max_attempts=max_attempts, ) - except: # noqa: E722 - state = self._get_node_state() - - if state.node_status == NodeStatus.Running: - assert type(state.pid) is int + except BaseException as exc1: + internal_utils.send_log_debug( + "Failed to wait for node initialization. {}".format( + ErrorUtils.ExceptionToHumanText(exc1) + ) + ) + try: self.stop() - else: - self._manually_started_pm_pid = state.pid - + except BaseException as exc2: + internal_utils.send_log_debug( + "Failed to stop a node after initialization failure. {}".format( + ErrorUtils.ExceptionToHumanText(exc2) + ) + ) + # Try to stabilization our internal state + self._manually_started_pm_pid = self._get_node_state().pid + raise raise return diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4aa52a9a..d6da71a4 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -525,14 +525,10 @@ def helper_step_of_failure_during_slow_start( node.slow_start(max_attempts=10) TestServices.PrintExceptionOK(x.value) - - assert not node.is_started - assert node.status() == NodeStatus.Stopped pass assert x is not None assert isinstance(x.value, Exception) - assert x.value.__context__ is None if type(x.value) is StartNodeException: assert x.value.description == "Cannot detect postmaster pid." @@ -543,6 +539,9 @@ def helper_step_of_failure_during_slow_start( assert "Connection reset by peer" in str(x.value) return False + assert not node.is_started + assert node.status() == NodeStatus.Stopped + assert type(x.value) is QueryTimeoutException return True From ca00a9e7471a67f38688adf01f616896488430b0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 15 Oct 2026 13:40:52 +0300 Subject: [PATCH 2/3] StopNodeException is added Also: - New class NodeException is a base class for Start/Stop node exceptions --- src/__init__.py | 2 + src/exceptions.py | 10 +++- src/node.py | 5 ++ tests/test_testgres_common.py | 12 ++++- .../exceptions/StopNodeException/__init__.py | 0 .../test_set001__constructor.py | 52 +++++++++++++++++++ 6 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 tests/units/exceptions/StopNodeException/__init__.py create mode 100644 tests/units/exceptions/StopNodeException/test_set001__constructor.py diff --git a/src/__init__.py b/src/__init__.py index b1ac160f..47db685a 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -24,6 +24,7 @@ TimeoutException, \ CatchUpException, \ StartNodeException, \ + StopNodeException, \ InitNodeException, \ BackupException, \ InvalidOperationException @@ -73,6 +74,7 @@ "QueryTimeoutException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", + "StopNodeException", "NodeApp", "PostgresNode", "PortManager", diff --git a/src/exceptions.py b/src/exceptions.py index d00de700..2bd47bc6 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -160,7 +160,7 @@ def __repr__(self) -> str: return result -class StartNodeException(TestgresException): +class NodeException(TestgresException): _description: typing.Optional[str] _files: typing.Optional[typing.Iterable] @@ -224,6 +224,14 @@ def __repr__(self) -> str: return result +class StartNodeException(NodeException): + pass + + +class StopNodeException(NodeException): + pass + + class InitNodeException(TestgresException): _message: typing.Optional[str] diff --git a/src/node.py b/src/node.py index c6190060..c3970329 100644 --- a/src/node.py +++ b/src/node.py @@ -46,6 +46,7 @@ QueryException, \ QueryTimeoutException, \ StartNodeException, \ + StopNodeException, \ TimeoutException, \ InitNodeException, \ TestgresException, \ @@ -1410,6 +1411,10 @@ def stop( logfile=self.utils_log_file, ) self._manually_started_pm_pid = None + except ExecUtilException as e: + msg = 'Cannot stop node' + files = self._collect_special_files() + raise StopNodeException(msg, files) from e finally: # always stop the reader thread, even if pg_ctl stop failed, # so it can't leak into interpreter shutdown. diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d6da71a4..044e02b0 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -38,6 +38,7 @@ from src import InitNodeException from src import StartNodeException +from src import StopNodeException from src import QueryException from src import ExecUtilException from src import QueryTimeoutException @@ -539,6 +540,9 @@ def helper_step_of_failure_during_slow_start( assert "Connection reset by peer" in str(x.value) return False + if type(x.value) is StopNodeException: + return True + assert not node.is_started assert node.status() == NodeStatus.Stopped @@ -642,7 +646,11 @@ def test_double_stop(self, node_svc: PostgresNodeService): node.stop() assert x is not None - assert "Is server running?" in str(x.value) + e1 = x.value + assert type(e1) is StopNodeException + e2 = e1.__cause__ + assert type(e2) is ExecUtilException + assert "Is server running?" in str(e2) assert not node.is_started @@ -2038,7 +2046,7 @@ def impl__test_pg_ctl_wait_option( try: node.stop(wait=False) break - except ExecUtilException as e: + except StopNodeException as e: # it's ok to get this exception here since node # could be not started yet logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) diff --git a/tests/units/exceptions/StopNodeException/__init__.py b/tests/units/exceptions/StopNodeException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/StopNodeException/test_set001__constructor.py b/tests/units/exceptions/StopNodeException/test_set001__constructor.py new file mode 100644 index 00000000..8ce9afe8 --- /dev/null +++ b/tests/units/exceptions/StopNodeException/test_set001__constructor.py @@ -0,0 +1,52 @@ +from src.exceptions import StopNodeException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = StopNodeException() + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert e.description is None + assert e.files is None + assert str(e) == "" + assert repr(e) == "StopNodeException()" + return + + def test_002__message(self): + e = StopNodeException(message="abc\n123") + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert e.description == "abc\n123" + assert e.files is None + assert str(e) == "abc\n123" + assert repr(e) == "StopNodeException(message='abc\\n123')" + return + + def test_003__files(self): + e = StopNodeException(files=[("f\n1", b'line1\nline2')]) + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "f\n1\n----\nb'line1\\nline2'\n" + assert e.description is None + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "f\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StopNodeException(files=[('f\\n1', b'line1\\nline2')])" + return + + def test_004__all(self): + e = StopNodeException(message="mmm", files=[("f\n1", b'line1\nline2')]) + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert e.description == "mmm" + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StopNodeException(message='mmm', files=[('f\\n1', b'line1\\nline2')])" + return From cb1d2e94a99fdc4de3e455c3128115607a45a0c8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 15 Oct 2026 16:53:13 +0300 Subject: [PATCH 3/3] fix: helper_step_of_failure_during_slow_start We have to work with our node within 'with' section! --- tests/test_testgres_common.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 044e02b0..058aec1d 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -525,29 +525,28 @@ def helper_step_of_failure_during_slow_start( with pytest.raises(expected_exception=Exception) as x: node.slow_start(max_attempts=10) - TestServices.PrintExceptionOK(x.value) - pass + assert x is not None + assert isinstance(x.value, Exception) - assert x is not None - assert isinstance(x.value, Exception) + TestServices.PrintExceptionOK(x.value) - if type(x.value) is StartNodeException: - assert x.value.description == "Cannot detect postmaster pid." - return False + if type(x.value) is StartNodeException: + assert x.value.description == "Cannot detect postmaster pid." + return False - if type(x.value) is ConnectionResetError: - # pg8000 - assert "Connection reset by peer" in str(x.value) - return False + if type(x.value) is ConnectionResetError: + # pg8000 + assert "Connection reset by peer" in str(x.value) + return False - if type(x.value) is StopNodeException: - return True + if type(x.value) is StopNodeException: + return True - assert not node.is_started - assert node.status() == NodeStatus.Stopped + assert not node.is_started + assert node.status() == NodeStatus.Stopped - assert type(x.value) is QueryTimeoutException - return True + assert type(x.value) is QueryTimeoutException + return True def test_restart(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService)