diff --git a/src/__init__.py b/src/__init__.py index b1ac160f..47db685a 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -24,6 +24,7 @@ TimeoutException, \ CatchUpException, \ StartNodeException, \ + StopNodeException, \ InitNodeException, \ BackupException, \ InvalidOperationException @@ -73,6 +74,7 @@ "QueryTimeoutException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", + "StopNodeException", "NodeApp", "PostgresNode", "PortManager", diff --git a/src/exceptions.py b/src/exceptions.py index d00de700..2bd47bc6 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -160,7 +160,7 @@ def __repr__(self) -> str: return result -class StartNodeException(TestgresException): +class NodeException(TestgresException): _description: typing.Optional[str] _files: typing.Optional[typing.Iterable] @@ -224,6 +224,14 @@ def __repr__(self) -> str: return result +class StartNodeException(NodeException): + pass + + +class StopNodeException(NodeException): + pass + + class InitNodeException(TestgresException): _message: typing.Optional[str] diff --git a/src/impl/error_utils.py b/src/impl/error_utils.py new file mode 100644 index 00000000..f099afb6 --- /dev/null +++ b/src/impl/error_utils.py @@ -0,0 +1,113 @@ +import traceback +import typing +import logging + + +class ErrorUtils: + @staticmethod + def ExceptionToHumanText(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if __class__.Helper__GetPrevExc(exc) is None: + return __class__.Helper__ExceptionToHumanText__Single(exc) + + return __class__.Helper__ExceptionToHumanText__Chain(exc) + + # -------------------------------------------------------------------- + @staticmethod + def Helper__ExceptionToHumanText__Single(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if isinstance(exc, AssertionError): + err_msg = "Exception ({}).".format(type(exc).__name__) + assert type(err_msg) is str + + exc_info_lines = traceback.format_exception(exc) + assert type(exc_info_lines) is list + err_msg2 = "".join(exc_info_lines).strip() + + if err_msg2 != "": + err_msg += " " + err_msg2 + + assert type(err_msg) is str + return err_msg + + err_msg = "Exception ({})".format(type(exc).__name__) + assert type(err_msg) is str + + err_msg2 = str(exc).strip() + + if err_msg2 == "": + err_msg += "." + else: + err_msg += ": " + err_msg2 + + assert type(err_msg) is str + return err_msg + + # -------------------------------------------------------------------- + @staticmethod + def Helper__ExceptionToHumanText__Chain(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + chain: typing.List[BaseException] = [] + curr = exc + + processed_exc_ids: typing.Set[int] = set() + + while curr is not None: + assert isinstance(curr, BaseException) + if id(curr) in processed_exc_ids: + logging.error("Cycle in exception chain: {}".format( + " --> ".join( + [type(x).__name__ for x in reversed(chain)], + ) + )) + break + + processed_exc_ids.add(id(curr)) + + chain.append(curr) + curr = __class__.Helper__GetPrevExc(curr) + continue + + lines: typing.List[str] = [] + + lines.append("It is a chain of exceptions (len: {}):".format( + len(chain), + )) + + n = 0 + for e in reversed(chain): + n += 1 + line1 = "---- {}. Exception ({})".format(n, type(e).__name__) + line2 = __class__.Helper__GetExcMsg(e) + + if line2 == "": + lines.append(line1) + else: + lines.append(line1 + ":") + lines.append(line2) + continue + + lines.append("--------") + + return "\n".join(lines) + + # -------------------------------------------------------------------- + @staticmethod + def Helper__GetPrevExc(exc: BaseException) -> typing.Optional[BaseException]: + assert isinstance(exc, BaseException) + return exc.__cause__ or exc.__context__ + + # -------------------------------------------------------------------- + @staticmethod + def Helper__GetExcMsg(exc: BaseException) -> str: + assert isinstance(exc, BaseException) + + if isinstance(exc, AssertionError): + exc_info_lines = traceback.format_exception(exc) + assert type(exc_info_lines) is list + return "".join(exc_info_lines).strip() + + return str(exc) diff --git a/src/node.py b/src/node.py index 4706a287..c3970329 100644 --- a/src/node.py +++ b/src/node.py @@ -46,6 +46,7 @@ QueryException, \ QueryTimeoutException, \ StartNodeException, \ + StopNodeException, \ TimeoutException, \ InitNodeException, \ TestgresException, \ @@ -55,6 +56,7 @@ from .port_manager import PortManager from .impl.port_manager__this_host import PortManager__ThisHost from .impl.port_manager__generic2 import PortManager__Generic2 +from .impl.error_utils import ErrorUtils from .impl import internal_utils from .logger import TestgresLogger @@ -1183,15 +1185,23 @@ def slow_start( suppress=suppressed_exceptions, max_attempts=max_attempts, ) - except: # noqa: E722 - state = self._get_node_state() - - if state.node_status == NodeStatus.Running: - assert type(state.pid) is int + except BaseException as exc1: + internal_utils.send_log_debug( + "Failed to wait for node initialization. {}".format( + ErrorUtils.ExceptionToHumanText(exc1) + ) + ) + try: self.stop() - else: - self._manually_started_pm_pid = state.pid - + except BaseException as exc2: + internal_utils.send_log_debug( + "Failed to stop a node after initialization failure. {}".format( + ErrorUtils.ExceptionToHumanText(exc2) + ) + ) + # Try to stabilization our internal state + self._manually_started_pm_pid = self._get_node_state().pid + raise raise return @@ -1401,6 +1411,10 @@ def stop( logfile=self.utils_log_file, ) self._manually_started_pm_pid = None + except ExecUtilException as e: + msg = 'Cannot stop node' + files = self._collect_special_files() + raise StopNodeException(msg, files) from e finally: # always stop the reader thread, even if pg_ctl stop failed, # so it can't leak into interpreter shutdown. diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4aa52a9a..058aec1d 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -38,6 +38,7 @@ from src import InitNodeException from src import StartNodeException +from src import StopNodeException from src import QueryException from src import ExecUtilException from src import QueryTimeoutException @@ -524,27 +525,28 @@ def helper_step_of_failure_during_slow_start( with pytest.raises(expected_exception=Exception) as x: node.slow_start(max_attempts=10) + assert x is not None + assert isinstance(x.value, Exception) + TestServices.PrintExceptionOK(x.value) - assert not node.is_started - assert node.status() == NodeStatus.Stopped - pass + if type(x.value) is StartNodeException: + assert x.value.description == "Cannot detect postmaster pid." + return False - assert x is not None - assert isinstance(x.value, Exception) - assert x.value.__context__ is None + if type(x.value) is ConnectionResetError: + # pg8000 + assert "Connection reset by peer" in str(x.value) + return False - if type(x.value) is StartNodeException: - assert x.value.description == "Cannot detect postmaster pid." - return False + if type(x.value) is StopNodeException: + return True - if type(x.value) is ConnectionResetError: - # pg8000 - assert "Connection reset by peer" in str(x.value) - return False + assert not node.is_started + assert node.status() == NodeStatus.Stopped - assert type(x.value) is QueryTimeoutException - return True + assert type(x.value) is QueryTimeoutException + return True def test_restart(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -643,7 +645,11 @@ def test_double_stop(self, node_svc: PostgresNodeService): node.stop() assert x is not None - assert "Is server running?" in str(x.value) + e1 = x.value + assert type(e1) is StopNodeException + e2 = e1.__cause__ + assert type(e2) is ExecUtilException + assert "Is server running?" in str(e2) assert not node.is_started @@ -2039,7 +2045,7 @@ def impl__test_pg_ctl_wait_option( try: node.stop(wait=False) break - except ExecUtilException as e: + except StopNodeException as e: # it's ok to get this exception here since node # could be not started yet logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) diff --git a/tests/units/exceptions/StopNodeException/__init__.py b/tests/units/exceptions/StopNodeException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/StopNodeException/test_set001__constructor.py b/tests/units/exceptions/StopNodeException/test_set001__constructor.py new file mode 100644 index 00000000..8ce9afe8 --- /dev/null +++ b/tests/units/exceptions/StopNodeException/test_set001__constructor.py @@ -0,0 +1,52 @@ +from src.exceptions import StopNodeException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = StopNodeException() + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert e.description is None + assert e.files is None + assert str(e) == "" + assert repr(e) == "StopNodeException()" + return + + def test_002__message(self): + e = StopNodeException(message="abc\n123") + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert e.description == "abc\n123" + assert e.files is None + assert str(e) == "abc\n123" + assert repr(e) == "StopNodeException(message='abc\\n123')" + return + + def test_003__files(self): + e = StopNodeException(files=[("f\n1", b'line1\nline2')]) + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "f\n1\n----\nb'line1\\nline2'\n" + assert e.description is None + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "f\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StopNodeException(files=[('f\\n1', b'line1\\nline2')])" + return + + def test_004__all(self): + e = StopNodeException(message="mmm", files=[("f\n1", b'line1\nline2')]) + assert type(e) is StopNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert e.description == "mmm" + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StopNodeException(message='mmm', files=[('f\\n1', b'line1\\nline2')])" + return