diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f610a70..fb62c651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.47.0 ### Features +* Expose the verdict the API declares on an error as typed attributes on every HTTP error: `retryable`, `error_type`, `error_reason` and `audience`. Errors the API raises deliberately publish these fields next to `detail`, eg a 503 OCR timeout with `"retryable": false`. Callers no longer need to parse `err.body` to read them. Each attribute is `None` when the body is not JSON, the field is absent, or its value has the wrong type. The retry loop reads `retryable` with the same parser. * Add `skip_preflight` to `CreateWorkflow` and `UpdateWorkflow`. Set it to `True` to skip the job preflight check for a workflow's jobs; on update, leaving it unset keeps the current value. `WorkflowInformation.skip_preflight` reports the workflow's current setting. For a one-off job, set `"skip_preflight": true` in the `request_data` JSON passed to `jobs.create_job`. ### Fixes diff --git a/README.md b/README.md index 897e659e..ed038cef 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,10 @@ with UnstructuredClient( | `err.body` | `str` | HTTP body. Can be empty string if no body is returned. | | `err.raw_response` | `httpx.Response` | Raw HTTP response | | `err.data` | | Optional. Some errors may contain structured data. [See Error Classes](#error-classes). | +| `err.retryable` | `Optional[bool]` | Whether the API declared that sending the same request again can succeed. `None` if it declared nothing. | +| `err.error_type` | `Optional[str]` | The kind of failure the API declared, eg `timeout`. `None` if it declared nothing. | +| `err.error_reason` | `Optional[str]` | A stable name for the failure the API declared. `None` if it declared nothing. | +| `err.audience` | `Optional[str]` | Whose fault the API declared the failure to be, `user` or `platform`. `None` if it declared nothing. | ### Example ```python diff --git a/_test_unstructured_client/unit/test_declared_errors.py b/_test_unstructured_client/unit/test_declared_errors.py new file mode 100644 index 00000000..e3cd2937 --- /dev/null +++ b/_test_unstructured_client/unit/test_declared_errors.py @@ -0,0 +1,148 @@ +import asyncio +import json + +import httpx +import pytest + +from unstructured_client import UnstructuredClient +from unstructured_client.models.errors import ( + HTTPValidationError, + HTTPValidationErrorData, + ResponseValidationError, + SDKError, + ServerError, + ServerErrorData, +) +from unstructured_client.utils import BackoffStrategy, RetryConfig +from unstructured_client.utils.declared_errors import ( + DeclaredErrorFields, + parse_declared_error_fields, +) + +DECLARED_BODY = { + "detail": "OCR timed out", + "retryable": False, + "error_type": "timeout", + "error_reason": "ocr_timeout", + "audience": "platform", +} +DECLARED_FIELDS = DeclaredErrorFields( + retryable=False, error_type="timeout", error_reason="ocr_timeout", audience="platform" +) + + +def _response(status_code: int, **kwargs) -> httpx.Response: + return httpx.Response( + status_code, + request=httpx.Request("POST", "http://test.invalid/general/v0/general"), + **kwargs, + ) + + +def _declared(error) -> DeclaredErrorFields: + return DeclaredErrorFields( + retryable=error.retryable, + error_type=error.error_type, + error_reason=error.error_reason, + audience=error.audience, + ) + + +@pytest.mark.parametrize( + "text,expected", + [ + ('{"detail":"x","retryable":false,"error_type":"timeout",' + '"error_reason":"ocr_timeout","audience":"platform"}', DECLARED_FIELDS), + ('{"retryable":true}', DeclaredErrorFields(retryable=True)), + ('{"detail":"x","retryable":false,"audience":null}', DeclaredErrorFields(retryable=False)), + ('{"detail":"Internal server error"}', DeclaredErrorFields()), + ('{"retryable":"false","error_type":5,"error_reason":" ","audience":[]}', + DeclaredErrorFields()), + ('{"retryable":0}', DeclaredErrorFields()), + ('[{"retryable":false}]', DeclaredErrorFields()), + ("false", DeclaredErrorFields()), + ("{broken", DeclaredErrorFields()), + ("upstream error", DeclaredErrorFields()), + ("", DeclaredErrorFields()), + (None, DeclaredErrorFields()), + pytest.param( + '{"retryable":false,"detail":' + "[" * 100_000 + "]" * 100_000 + "}", + DeclaredErrorFields(), + id="deeply-nested", + ), + ], +) +def test_parse_declared_error_fields(text, expected): + assert parse_declared_error_fields(text) == expected + + +@pytest.mark.parametrize( + "build", + [ + lambda r: ServerError(ServerErrorData(detail="OCR timed out"), r), + lambda r: SDKError("API error occurred", r, r.text), + lambda r: HTTPValidationError(HTTPValidationErrorData(detail="OCR timed out"), r), + ], + ids=["ServerError", "SDKError", "HTTPValidationError"], +) +@pytest.mark.parametrize( + "content", + [ + json.dumps(DECLARED_BODY).encode(), + b"\xef\xbb\xbf" + json.dumps(DECLARED_BODY).encode(), + json.dumps(DECLARED_BODY).encode("utf-16"), + ], + ids=["utf-8", "utf-8-bom", "utf-16"], +) +def test_error_classes_expose_declared_fields(build, content): + response = _response(503, content=content, headers={"Content-Type": "application/json"}) + + assert _declared(build(response)) == DECLARED_FIELDS + + +def test_unread_stream_falls_back_to_body(): + class Stream(httpx.SyncByteStream): + def __iter__(self): + pytest.fail("error construction read an unread stream") + yield b"" + + response = _response(503, stream=Stream()) + error = SDKError("API error occurred", response, json.dumps(DECLARED_BODY)) + + assert _declared(error) == DECLARED_FIELDS + + +def test_success_response_declares_nothing(): + response = _response(200, json=DECLARED_BODY) + error = ResponseValidationError("Response validation failed", response, ValueError()) + + assert _declared(error) == DeclaredErrorFields() + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("status_code,error_class", [(400, SDKError), (503, ServerError)]) +def test_public_partition_raises_declared_fields(async_mode, status_code, error_class): + def respond(request): + return httpx.Response(status_code, json=DECLARED_BODY) + + transport = httpx.MockTransport(respond) + request = {"partition_parameters": { + "files": {"content": b"hello", "file_name": "test.txt"}, + "split_pdf_page": False, + }} + retries = RetryConfig("none", BackoffStrategy(1, 1, 1.0, 1), False) + + async def run_async(): + async with httpx.AsyncClient(transport=transport) as async_client: + sdk = UnstructuredClient(server_url="http://test.invalid", async_client=async_client) + await sdk.general.partition_async(request=request, retries=retries) + + with pytest.raises(error_class) as exc: + if async_mode: + asyncio.run(run_async()) + else: + with httpx.Client(transport=transport) as client: + sdk = UnstructuredClient(server_url="http://test.invalid", client=client) + sdk.general.partition(request=request, retries=retries) + + assert _declared(exc.value) == DECLARED_FIELDS diff --git a/_test_unstructured_client/unit/test_retries.py b/_test_unstructured_client/unit/test_retries.py index 6419b9d4..bce7175c 100644 --- a/_test_unstructured_client/unit/test_retries.py +++ b/_test_unstructured_client/unit/test_retries.py @@ -599,6 +599,8 @@ async def run(): (b'upstream error', "text/plain", False), (b'{"retryable":false}', "text/plain", True), (b'{"retryable":false}', "", True), + (b'\xef\xbb\xbf{"retryable":false}', "application/json", True), + ('{"retryable":false}'.encode("utf-16"), "application/json", True), ], ) def test_declared_retry_verdict( diff --git a/src/unstructured_client/models/errors/unstructuredclienterror.py b/src/unstructured_client/models/errors/unstructuredclienterror.py index 054fb875..ac8ee9c1 100644 --- a/src/unstructured_client/models/errors/unstructuredclienterror.py +++ b/src/unstructured_client/models/errors/unstructuredclienterror.py @@ -1,7 +1,12 @@ """Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" import httpx -from typing import Optional +from typing import Optional, Union + +from unstructured_client.utils.declared_errors import ( + DeclaredErrorFields, + parse_declared_error_fields, +) class UnstructuredClientError(Exception): @@ -12,6 +17,10 @@ class UnstructuredClientError(Exception): body: str headers: httpx.Headers raw_response: httpx.Response + retryable: Optional[bool] + error_type: Optional[str] + error_reason: Optional[str] + audience: Optional[str] def __init__( self, message: str, raw_response: httpx.Response, body: Optional[str] = None @@ -22,5 +31,24 @@ def __init__( self.headers = raw_response.headers self.raw_response = raw_response + # A success response that failed validation can carry a large body that + # declares nothing, so only error statuses are parsed. + declared = ( + parse_declared_error_fields(self._raw_body()) + if self.status_code >= 400 + else DeclaredErrorFields() + ) + self.retryable = declared.retryable + self.error_type = declared.error_type + self.error_reason = declared.error_reason + self.audience = declared.audience + + def _raw_body(self) -> Union[str, bytes]: + # The response bytes keep the JSON encoding detection that decoded text loses. + try: + return self.raw_response.content + except httpx.ResponseNotRead: + return self.body + def __str__(self): return self.message diff --git a/src/unstructured_client/utils/declared_errors.py b/src/unstructured_client/utils/declared_errors.py new file mode 100644 index 00000000..64af26fb --- /dev/null +++ b/src/unstructured_client/utils/declared_errors.py @@ -0,0 +1,47 @@ +"""The verdict the Unstructured API declares beside ``detail`` in an error body. + +Errors the API raises deliberately carry ``retryable``, ``error_type``, +``error_reason`` and ``audience`` next to ``detail``. Errors it did not classify +carry only ``detail``, and every field reads as ``None``. +""" + +import json +from dataclasses import dataclass +from typing import Any, Optional, Union + + +@dataclass(frozen=True) +class DeclaredErrorFields: + retryable: Optional[bool] = None + error_type: Optional[str] = None + error_reason: Optional[str] = None + audience: Optional[str] = None + + +def _non_empty_str(value: Any) -> Optional[str]: + return value if isinstance(value, str) and value.strip() else None + + +def _declared_error_fields(body: Any) -> DeclaredErrorFields: + if not isinstance(body, dict): + return DeclaredErrorFields() + retryable = body.get("retryable") + return DeclaredErrorFields( + retryable=retryable if isinstance(retryable, bool) else None, + error_type=_non_empty_str(body.get("error_type")), + error_reason=_non_empty_str(body.get("error_reason")), + audience=_non_empty_str(body.get("audience")), + ) + + +def parse_declared_error_fields(raw: Union[str, bytes, None]) -> DeclaredErrorFields: + """Read the declared fields from a raw body, dropping any of the wrong type. + + Bytes are decoded as JSON, which detects UTF-8, UTF-16 and UTF-32 and skips a + UTF-8 byte order mark. A body that is not a JSON object declares nothing. + """ + try: + body = json.loads(raw) if raw else None + except (ValueError, TypeError, RecursionError): + return DeclaredErrorFields() + return _declared_error_fields(body) diff --git a/src/unstructured_client/utils/retries.py b/src/unstructured_client/utils/retries.py index d268fefb..6d4e4f67 100644 --- a/src/unstructured_client/utils/retries.py +++ b/src/unstructured_client/utils/retries.py @@ -7,6 +7,8 @@ import httpx +from unstructured_client.utils.declared_errors import parse_declared_error_fields + class BackoffStrategy: initial_interval: int @@ -90,10 +92,10 @@ def __init__(self, inner: Exception): def _declares_non_retryable(response: httpx.Response) -> bool: # Inspect buffered error bodies without consuming unread response streams. try: - body = response.json() - except (ValueError, RecursionError, httpx.ResponseNotRead): + content = response.content + except httpx.ResponseNotRead: return False - return isinstance(body, dict) and body.get("retryable") is False + return parse_declared_error_fields(content).retryable is False def _is_retry_status(response: httpx.Response, status_codes: List[str]) -> bool: